diff options
author | Kamil Trzciński <ayufan@ayufan.eu> | 2016-05-18 18:19:01 +0000 |
---|---|---|
committer | Kamil Trzciński <ayufan@ayufan.eu> | 2016-05-18 18:19:01 +0000 |
commit | 08fddae7441ce9ac61bc80e5b93a2eafb8441430 (patch) | |
tree | 3921871fc9363f9ef869e6b6b36963e91446bd9e | |
parent | b7d83acf5b03e08dc9e387e1abb83c5e3c80444c (diff) | |
parent | 2bea20b8ef4bb4c0577bfe68bb77f96958e164d2 (diff) | |
download | gitlab-ce-08fddae7441ce9ac61bc80e5b93a2eafb8441430.tar.gz |
Merge branch 'docker-registry-view' into 'master'
Add container registry support
Tasks:
- [x] Merge docker/distribution authentication service: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3787
- [x] Implement Docker Registry API
- [x] Show a list of docker images in GitLab
- [x] Remove registry repository on project deletion
- [x] Support project rename, move and namespace rename
- [x] Use token when connecting the registry
- [x] Allow to delete images from GitLab
- [x] Support pushing from GitLab CI (gitlab-ci-token / $CI_BUILD_TOKEN)
- [x] Support GitLab Runner pulling for public repositories
- [ ] Support GitLab Runner pulling for private repositories
- [x] Add tests for Docker Registry API
- [x] Add tests for a views
- [x] Make texts nicer
- [x] Implement a backup support
- [ ] Create administration documentation https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/4141
- [ ] Create user documentation
See merge request !4040
41 files changed, 954 insertions, 22 deletions
diff --git a/app/controllers/projects/container_registry_controller.rb b/app/controllers/projects/container_registry_controller.rb new file mode 100644 index 00000000000..d1f46497207 --- /dev/null +++ b/app/controllers/projects/container_registry_controller.rb @@ -0,0 +1,34 @@ +class Projects::ContainerRegistryController < Projects::ApplicationController + before_action :verify_registry_enabled + before_action :authorize_read_container_image! + before_action :authorize_update_container_image!, only: [:destroy] + layout 'project' + + def index + @tags = container_registry_repository.tags + end + + def destroy + url = namespace_project_container_registry_index_path(project.namespace, project) + + if tag.delete + redirect_to url + else + redirect_to url, alert: 'Failed to remove tag' + end + end + + private + + def verify_registry_enabled + render_404 unless Gitlab.config.registry.enabled + end + + def container_registry_repository + @container_registry_repository ||= project.container_registry_repository + end + + def tag + @tag ||= container_registry_repository.tag(params[:id]) + end +end diff --git a/app/helpers/gitlab_routing_helper.rb b/app/helpers/gitlab_routing_helper.rb index f07eff3fb57..2ce2d4e694f 100644 --- a/app/helpers/gitlab_routing_helper.rb +++ b/app/helpers/gitlab_routing_helper.rb @@ -33,6 +33,10 @@ module GitlabRoutingHelper namespace_project_builds_path(project.namespace, project, *args) end + def project_container_registry_path(project, *args) + namespace_project_container_registry_index_path(project.namespace, project, *args) + end + def activity_project_path(project, *args) activity_namespace_project_path(project.namespace, project, *args) end diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index e1ab78df69e..a140bbc3c67 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -152,6 +152,10 @@ module ProjectsHelper nav_tabs << :builds end + if Gitlab.config.registry.enabled && can?(current_user, :read_container_image, project) + nav_tabs << :container_registry + end + if can?(current_user, :admin_project, project) nav_tabs << :settings end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 9c942a8f4e3..da19462f265 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -110,6 +110,10 @@ class Namespace < ActiveRecord::Base # Ensure old directory exists before moving it gitlab_shell.add_namespace(path_was) + if any_project_has_container_registry_tags? + raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry') + end + if gitlab_shell.mv_namespace(path_was, path) Gitlab::UploadsTransfer.new.rename_namespace(path_was, path) @@ -131,6 +135,10 @@ class Namespace < ActiveRecord::Base end end + def any_project_has_container_registry_tags? + projects.any?(&:has_container_registry_tags?) + end + def send_update_instructions projects.each do |project| project.send_move_instructions("#{path_was}/#{project.path}") diff --git a/app/models/project.rb b/app/models/project.rb index b7ae34c95c6..2a6b0839b64 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -329,12 +329,30 @@ class Project < ActiveRecord::Base @repository ||= Repository.new(path_with_namespace, self) end - def container_registry_url - if container_registry_enabled? && Gitlab.config.registry.enabled - "#{Gitlab.config.registry.host_with_port}/#{path_with_namespace}" + def container_registry_repository + return unless Gitlab.config.registry.enabled + + @container_registry_repository ||= begin + token = Auth::ContainerRegistryAuthenticationService.full_access_token(path_with_namespace) + url = Gitlab.config.registry.api_url + host_port = Gitlab.config.registry.host_port + registry = ContainerRegistry::Registry.new(url, token: token, path: host_port) + registry.repository(path_with_namespace) end end + def container_registry_repository_url + if Gitlab.config.registry.enabled + "#{Gitlab.config.registry.host_port}/#{path_with_namespace}" + end + end + + def has_container_registry_tags? + return unless container_registry_repository + + container_registry_repository.tags.any? + end + def commit(id = 'HEAD') repository.commit(id) end @@ -746,6 +764,11 @@ class Project < ActiveRecord::Base expire_caches_before_rename(old_path_with_namespace) + if has_container_registry_tags? + # we currently doesn't support renaming repository if it contains tags in container registry + raise Exception.new('Project cannot be renamed, because tags are present in its container registry') + end + if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace) # If repository moved successfully we need to send update instructions to users. # However we cannot allow rollback since we moved repository diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index b636f55d031..3144e96ba31 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -6,7 +6,7 @@ module Auth return error('not found', 404) unless registry.enabled if params[:offline_token] - return error('forbidden', 403) unless current_user + return error('unauthorized', 401) unless current_user else return error('forbidden', 403) unless scope end @@ -14,6 +14,17 @@ module Auth { token: authorized_token(scope).encoded } end + def self.full_access_token(*names) + registry = Gitlab.config.registry + token = JSONWebToken::RSAToken.new(registry.key) + token.issuer = registry.issuer + token.audience = AUDIENCE + token[:access] = names.map do |name| + { type: 'repository', name: name, actions: %w(pull push) } + end + token.encoded + end + private def authorized_token(*accesses) diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index 48a6131b444..f09072975c3 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -26,6 +26,10 @@ module Projects Project.transaction do project.destroy! + unless remove_registry_tags + raise_error('Failed to remove project container registry. Please try again or contact administrator') + end + unless remove_repository(repo_path) raise_error('Failed to remove project repository. Please try again or contact administrator') end @@ -59,6 +63,12 @@ module Projects end end + def remove_registry_tags + return true unless Gitlab.config.registry.enabled + + project.container_registry_repository.delete_tags + end + def raise_error(message) raise DestroyError.new(message) end diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 111b3ec05ea..03b57dea51e 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -34,6 +34,11 @@ module Projects raise TransferError.new("Project with same path in target namespace already exists") end + if project.has_container_registry_tags? + # we currently doesn't support renaming repository if it contains tags in container registry + raise TransferError.new('Project cannot be transferred, because tags are present in its container registry') + end + project.expire_caches_before_rename(old_path) # Apply new namespace id and visibility level diff --git a/app/views/layouts/nav/_project.html.haml b/app/views/layouts/nav/_project.html.haml index 479bde33719..d3d715aad3b 100644 --- a/app/views/layouts/nav/_project.html.haml +++ b/app/views/layouts/nav/_project.html.haml @@ -46,6 +46,13 @@ Builds %span.count.builds_counter= number_with_delimiter(@project.builds.running_or_pending.count(:all)) + - if project_nav_tab? :container_registry + = nav_link(controller: %w(container_registry)) do + = link_to project_container_registry_path(@project), title: 'Container Registry', class: 'shortcuts-container-registry' do + = icon('hdd-o fw') + %span + Container Registry + - if project_nav_tab? :graphs = nav_link(controller: %w(graphs)) do = link_to namespace_project_graph_path(@project.namespace, @project, current_ref), title: 'Graphs', class: 'shortcuts-graphs' do diff --git a/app/views/projects/container_registry/_header_title.html.haml b/app/views/projects/container_registry/_header_title.html.haml new file mode 100644 index 00000000000..f1863c52a3e --- /dev/null +++ b/app/views/projects/container_registry/_header_title.html.haml @@ -0,0 +1 @@ +- header_title project_title(@project, "Container Registry", project_container_registry_path(@project)) diff --git a/app/views/projects/container_registry/_tag.html.haml b/app/views/projects/container_registry/_tag.html.haml new file mode 100644 index 00000000000..4e9f936539b --- /dev/null +++ b/app/views/projects/container_registry/_tag.html.haml @@ -0,0 +1,21 @@ +%tr.tag + %td + = escape_once(tag.name) + = clipboard_button(clipboard_text: "docker pull #{tag.path}") + %td + - if layer = tag.layers.first + %span.has-tooltip{ title: "#{layer.revision}" } + = layer.short_revision + - else + \- + %td + = number_to_human_size(tag.total_size) + · + = pluralize(tag.layers.size, "layer") + %td + = time_ago_in_words(tag.created_at) + - if can?(current_user, :update_container_image, @project) + %td.content + .controls.hidden-xs.pull-right + = link_to namespace_project_container_registry_path(@project.namespace, @project, tag.name), class: 'btn btn-remove has-tooltip', title: "Remove", data: { confirm: "Are you sure?" }, method: :delete do + = icon("trash cred") diff --git a/app/views/projects/container_registry/index.html.haml b/app/views/projects/container_registry/index.html.haml new file mode 100644 index 00000000000..40957993b22 --- /dev/null +++ b/app/views/projects/container_registry/index.html.haml @@ -0,0 +1,40 @@ +- page_title "Container Registry" += render "header_title" + +%hr + +%ul.content-list + .light.prepend-top-default + %p + A 'container image' is a snapshot of a container. + You can host your container images with GitLab. + %br + To start using container images hosted on GitLab you first need to login: + %pre + %code + docker login #{Gitlab.config.registry.host_port} + %br + Then you are free to create and upload a container image with build and push commands: + %pre + docker build -t #{escape_once(@project.container_registry_repository_url)} . + %br + docker push #{escape_once(@project.container_registry_repository_url)} + + - if @tags.blank? + %li + .nothing-here-block No images in Container Registry for this project. + + - else + .table-holder + %table.table.tags + %thead + %tr + %th Name + %th Image ID + %th Size + %th Created + - if can?(current_user, :update_container_image, @project) + %th + + - @tags.each do |tag| + = render 'tag', tag: tag
\ No newline at end of file diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index d935121d88b..0cef2794f4e 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -183,6 +183,7 @@ production: &base # api_url: http://localhost:5000/ # key: config/registry.key # issuer: omnibus-certificate + # path: shared/registry # # 2. GitLab CI settings diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index d1fcb053bee..124d63ce3ac 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -249,9 +249,12 @@ Settings.artifacts['max_size'] ||= 100 # in megabytes Settings['registry'] ||= Settingslogic.new({}) Settings.registry['enabled'] ||= false Settings.registry['host'] ||= "example.com" +Settings.registry['port'] ||= nil Settings.registry['api_url'] ||= "http://localhost:5000/" Settings.registry['key'] ||= nil Settings.registry['issuer'] ||= nil +Settings.registry['host_port'] ||= [Settings.registry['host'], Settings.registry['port']].compact.join(':') +Settings.registry['path'] = File.expand_path(Settings.registry['path'] || File.join(Settings.shared['path'], 'registry'), Rails.root) # # Git LFS diff --git a/config/routes.rb b/config/routes.rb index e1b72556098..09264b5493e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -693,6 +693,8 @@ Rails.application.routes.draw do end end + resources :container_registry, only: [:index, :destroy], constraints: { id: Gitlab::Regex.container_registry_reference_regex } + resources :milestones, constraints: { id: /\d+/ } do member do put :sort_issues diff --git a/doc/permissions/permissions.md b/doc/permissions/permissions.md index 6be5ea0b486..b76ce31cbad 100644 --- a/doc/permissions/permissions.md +++ b/doc/permissions/permissions.md @@ -39,6 +39,7 @@ documentation](../workflow/add-user/add-user.md). | Cancel and retry builds | | | ✓ | ✓ | ✓ | | Create or update commit status | | | ✓ | ✓ | ✓ | | Update a container registry | | | ✓ | ✓ | ✓ | +| Remove a container registry image | | | ✓ | ✓ | ✓ | | Create new milestones | | | | ✓ | ✓ | | Add new team members | | | | ✓ | ✓ | | Push to protected branches | | | | ✓ | ✓ | diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 4962f5e53ce..7d0608f09da 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -157,7 +157,7 @@ module Backup end def archives_to_backup - %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + %w{uploads builds artifacts lfs registry}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact end def folders_to_backup diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb new file mode 100644 index 00000000000..67fe0231087 --- /dev/null +++ b/lib/backup/registry.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Registry < Files + def initialize + super('registry', Settings.registry.path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb new file mode 100644 index 00000000000..4e20dc4f875 --- /dev/null +++ b/lib/container_registry/blob.rb @@ -0,0 +1,48 @@ +module ContainerRegistry + class Blob + attr_reader :repository, :config + + delegate :registry, :client, to: :repository + + def initialize(repository, config) + @repository = repository + @config = config || {} + end + + def valid? + digest.present? + end + + def path + "#{repository.path}@#{digest}" + end + + def digest + config['digest'] + end + + def type + config['mediaType'] + end + + def size + config['size'] + end + + def revision + digest.split(':')[1] + end + + def short_revision + revision[0..8] + end + + def delete + client.delete_blob(repository.name, digest) + end + + def data + @data ||= client.blob(repository.name, digest, type) + end + end +end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb new file mode 100644 index 00000000000..4d726692f45 --- /dev/null +++ b/lib/container_registry/client.rb @@ -0,0 +1,61 @@ +require 'faraday' +require 'faraday_middleware' + +module ContainerRegistry + class Client + attr_accessor :uri + + MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json' + + def initialize(base_uri, options = {}) + @base_uri = base_uri + @faraday = Faraday.new(@base_uri) do |conn| + initialize_connection(conn, options) + end + end + + def repository_tags(name) + @faraday.get("/v2/#{name}/tags/list").body + end + + def repository_manifest(name, reference) + @faraday.get("/v2/#{name}/manifests/#{reference}").body + end + + def repository_tag_digest(name, reference) + response = @faraday.head("/v2/#{name}/manifests/#{reference}") + response.headers['docker-content-digest'] if response.success? + end + + def delete_repository_tag(name, reference) + @faraday.delete("/v2/#{name}/manifests/#{reference}").success? + end + + def blob(name, digest, type = nil) + headers = {} + headers['Accept'] = type if type + @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers).body + end + + def delete_blob(name, digest) + @faraday.delete("/v2/#{name}/blobs/#{digest}").success? + end + + private + + def initialize_connection(conn, options) + conn.request :json + conn.headers['Accept'] = MANIFEST_VERSION + + conn.response :json, content_type: /\bjson$/ + + if options[:user] && options[:password] + conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) + elsif options[:token] + conn.request(:authorization, :bearer, options[:token].to_s) + end + + conn.adapter :net_http + end + end +end diff --git a/lib/container_registry/config.rb b/lib/container_registry/config.rb new file mode 100644 index 00000000000..589f9f4380a --- /dev/null +++ b/lib/container_registry/config.rb @@ -0,0 +1,16 @@ +module ContainerRegistry + class Config + attr_reader :tag, :blob, :data + + def initialize(tag, blob) + @tag, @blob = tag, blob + @data = JSON.parse(blob.data) + end + + def [](key) + return unless data + + data[key] + end + end +end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb new file mode 100644 index 00000000000..0e634f6b6ef --- /dev/null +++ b/lib/container_registry/registry.rb @@ -0,0 +1,21 @@ +module ContainerRegistry + class Registry + attr_reader :uri, :client, :path + + def initialize(uri, options = {}) + @uri = uri + @path = options[:path] || default_path + @client = ContainerRegistry::Client.new(uri, options) + end + + def repository(name) + ContainerRegistry::Repository.new(self, name) + end + + private + + def default_path + @uri.sub(/^https?:\/\//, '') + end + end +end diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb new file mode 100644 index 00000000000..0e4a7cb3cc9 --- /dev/null +++ b/lib/container_registry/repository.rb @@ -0,0 +1,48 @@ +module ContainerRegistry + class Repository + attr_reader :registry, :name + + delegate :client, to: :registry + + def initialize(registry, name) + @registry, @name = registry, name + end + + def path + [registry.path, name].compact.join('/') + end + + def tag(tag) + ContainerRegistry::Tag.new(self, tag) + end + + def manifest + return @manifest if defined?(@manifest) + + @manifest = client.repository_tags(name) + end + + def valid? + manifest.present? + end + + def tags + return @tags if defined?(@tags) + return [] unless manifest && manifest['tags'] + + @tags = manifest['tags'].map do |tag| + ContainerRegistry::Tag.new(self, tag) + end + end + + def blob(config) + ContainerRegistry::Blob.new(self, config) + end + + def delete_tags + return unless tags + + tags.all?(&:delete) + end + end +end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb new file mode 100644 index 00000000000..43f8d6dc8c2 --- /dev/null +++ b/lib/container_registry/tag.rb @@ -0,0 +1,77 @@ +module ContainerRegistry + class Tag + attr_reader :repository, :name + + delegate :registry, :client, to: :repository + + def initialize(repository, name) + @repository, @name = repository, name + end + + def valid? + manifest.present? + end + + def manifest + return @manifest if defined?(@manifest) + + @manifest = client.repository_manifest(repository.name, name) + end + + def path + "#{repository.path}:#{name}" + end + + def [](key) + return unless manifest + + manifest[key] + end + + def digest + return @digest if defined?(@digest) + + @digest = client.repository_tag_digest(repository.name, name) + end + + def config_blob + return @config_blob if defined?(@config_blob) + return unless manifest && manifest['config'] + + @config_blob = repository.blob(manifest['config']) + end + + def config + return unless config_blob + + @config ||= ContainerRegistry::Config.new(self, config_blob) + end + + def created_at + return unless config + + @created_at ||= DateTime.rfc3339(config['created']) + end + + def layers + return @layers if defined?(@layers) + return unless manifest + + @layers = manifest['layers'].map do |layer| + repository.blob(layer) + end + end + + def total_size + return unless layers + + layers.map(&:size).sum + end + + def delete + return unless digest + + client.delete_repository_tag(repository.name, digest) + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ace906a6f59..1cbd6d945a0 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -96,5 +96,9 @@ module Gitlab (?<![\/.]) (?# rule #6-7) }x.freeze end + + def container_registry_reference_regex + git_reference_regex + end end end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 402bb338f27..d97d974ec20 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -14,6 +14,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:artifacts:create"].invoke Rake::Task["gitlab:backup:lfs:create"].invoke + Rake::Task["gitlab:backup:registry:create"].invoke backup = Backup::Manager.new backup.pack @@ -54,6 +55,7 @@ namespace :gitlab do Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') + Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke backup.cleanup @@ -173,6 +175,25 @@ namespace :gitlab do end end + namespace :registry do + task create: :environment do + $progress.puts "Dumping container registry images ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("registry") + $progress.puts "[SKIPPED]".cyan + else + Backup::Registry.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring container registry images ... ".blue + Backup::Registry.new.restore + $progress.puts "done".green + end + end + def configure_cron_mode if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a diff --git a/shared/registry/.gitkeep b/shared/registry/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/shared/registry/.gitkeep diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb new file mode 100644 index 00000000000..53b4f027117 --- /dev/null +++ b/spec/features/container_registry_spec.rb @@ -0,0 +1,44 @@ +require 'spec_helper' + +describe "Container Registry" do + let(:project) { create(:empty_project) } + let(:repository) { project.container_registry_repository } + let(:tag_name) { 'latest' } + let(:tags) { [tag_name] } + + before do + login_as(:user) + project.team << [@user, :developer] + stub_container_registry_tags(*tags) + stub_container_registry_config(enabled: true) + allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') + end + + describe 'GET /:project/container_registry' do + before do + visit namespace_project_container_registry_index_path(project.namespace, project) + end + + context 'when no tags' do + let(:tags) { [] } + + it { expect(page).to have_content('No images in Container Registry for this project') } + end + + context 'when there are tags' do + it { expect(page).to have_content(tag_name)} + end + end + + describe 'DELETE /:project/container_registry/tag' do + before do + visit namespace_project_container_registry_index_path(project.namespace, project) + end + + it do + expect_any_instance_of(::ContainerRegistry::Tag).to receive(:delete).and_return(true) + + click_on 'Remove' + end + end +end diff --git a/spec/fixtures/container_registry/config_blob.json b/spec/fixtures/container_registry/config_blob.json new file mode 100644 index 00000000000..1028c994a24 --- /dev/null +++ b/spec/fixtures/container_registry/config_blob.json @@ -0,0 +1 @@ +{"architecture":"amd64","config":{"Hostname":"b14cd8298755","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"b14cd82987550b01af9a666a2f4c996280a6152e66873134fae5a0f223dc5976","container_config":{"Hostname":"b14cd8298755","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":["/bin/sh","-c","#(nop) ADD file:033ab063740d9ff4dcfb1c69eccf25f91d88729f57cd5a73050e014e3e094aa0 in /"],"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2016-04-01T20:53:00.160300546Z","docker_version":"1.9.1","history":[{"created":"2016-04-01T20:53:00.160300546Z","created_by":"/bin/sh -c #(nop) ADD file:033ab063740d9ff4dcfb1c69eccf25f91d88729f57cd5a73050e014e3e094aa0 in /"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:c56b7dabbc7aa730eeab07668bdcbd7e3d40855047ca9a0cc1bfed23a2486111"]}} diff --git a/spec/fixtures/container_registry/tag_manifest.json b/spec/fixtures/container_registry/tag_manifest.json new file mode 100644 index 00000000000..1b6008e2872 --- /dev/null +++ b/spec/fixtures/container_registry/tag_manifest.json @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/octet-stream","size":1145,"digest":"sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac"},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","size":2319870,"digest":"sha256:420890c9e918b6668faaedd9000e220190f2493b0693ee563ebd7b4cc754a57d"}]} diff --git a/spec/lib/container_registry/blob_spec.rb b/spec/lib/container_registry/blob_spec.rb new file mode 100644 index 00000000000..4d8cb787dde --- /dev/null +++ b/spec/lib/container_registry/blob_spec.rb @@ -0,0 +1,61 @@ +require 'spec_helper' + +describe ContainerRegistry::Blob do + let(:digest) { 'sha256:0123456789012345' } + let(:config) do + { + 'digest' => digest, + 'mediaType' => 'binary', + 'size' => 1000 + } + end + + let(:registry) { ContainerRegistry::Registry.new('http://example.com') } + let(:repository) { registry.repository('group/test') } + let(:blob) { repository.blob(config) } + + it { expect(blob).to respond_to(:repository) } + it { expect(blob).to delegate_method(:registry).to(:repository) } + it { expect(blob).to delegate_method(:client).to(:repository) } + + context '#path' do + subject { blob.path } + + it { is_expected.to eq('example.com/group/test@sha256:0123456789012345') } + end + + context '#digest' do + subject { blob.digest } + + it { is_expected.to eq(digest) } + end + + context '#type' do + subject { blob.type } + + it { is_expected.to eq('binary') } + end + + context '#revision' do + subject { blob.revision } + + it { is_expected.to eq('0123456789012345') } + end + + context '#short_revision' do + subject { blob.short_revision } + + it { is_expected.to eq('012345678') } + end + + context '#delete' do + before do + stub_request(:delete, 'http://example.com/v2/group/test/blobs/sha256:0123456789012345'). + to_return(status: 200) + end + + subject { blob.delete } + + it { is_expected.to be_truthy } + end +end diff --git a/spec/lib/container_registry/registry_spec.rb b/spec/lib/container_registry/registry_spec.rb new file mode 100644 index 00000000000..2638401ae6e --- /dev/null +++ b/spec/lib/container_registry/registry_spec.rb @@ -0,0 +1,28 @@ +require 'spec_helper' + +describe ContainerRegistry::Registry do + let(:path) { nil } + let(:registry) { described_class.new('http://example.com', path: path) } + + subject { registry } + + it { is_expected.to respond_to(:client) } + it { is_expected.to respond_to(:uri) } + it { is_expected.to respond_to(:path) } + + it { expect(subject.repository('test')).to_not be_nil } + + context '#path' do + subject { registry.path } + + context 'path from URL' do + it { is_expected.to eq('example.com') } + end + + context 'custom path' do + let(:path) { 'registry.example.com' } + + it { is_expected.to eq(path) } + end + end +end diff --git a/spec/lib/container_registry/repository_spec.rb b/spec/lib/container_registry/repository_spec.rb new file mode 100644 index 00000000000..e6d66b11e4e --- /dev/null +++ b/spec/lib/container_registry/repository_spec.rb @@ -0,0 +1,65 @@ +require 'spec_helper' + +describe ContainerRegistry::Repository do + let(:registry) { ContainerRegistry::Registry.new('http://example.com') } + let(:repository) { registry.repository('group/test') } + + it { expect(repository).to respond_to(:registry) } + it { expect(repository).to delegate_method(:client).to(:registry) } + it { expect(repository.tag('test')).to_not be_nil } + + context '#path' do + subject { repository.path } + + it { is_expected.to eq('example.com/group/test') } + end + + context 'manifest processing' do + before do + stub_request(:get, 'http://example.com/v2/group/test/tags/list'). + with(headers: { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' }). + to_return( + status: 200, + body: JSON.dump(tags: ['test']), + headers: { 'Content-Type' => 'application/vnd.docker.distribution.manifest.v2+json' }) + end + + context '#manifest' do + subject { repository.manifest } + + it { is_expected.to_not be_nil } + end + + context '#valid?' do + subject { repository.valid? } + + it { is_expected.to be_truthy } + end + + context '#tags' do + subject { repository.tags } + + it { is_expected.to_not be_empty } + end + end + + context '#delete_tags' do + let(:tag) { ContainerRegistry::Tag.new(repository, 'tag') } + + before { expect(repository).to receive(:tags).twice.and_return([tag]) } + + subject { repository.delete_tags } + + context 'succeeds' do + before { expect(tag).to receive(:delete).and_return(true) } + + it { is_expected.to be_truthy } + end + + context 'any fails' do + before { expect(tag).to receive(:delete).and_return(false) } + + it { is_expected.to be_falsey } + end + end +end diff --git a/spec/lib/container_registry/tag_spec.rb b/spec/lib/container_registry/tag_spec.rb new file mode 100644 index 00000000000..12cf91127ed --- /dev/null +++ b/spec/lib/container_registry/tag_spec.rb @@ -0,0 +1,89 @@ +require 'spec_helper' + +describe ContainerRegistry::Tag do + let(:registry) { ContainerRegistry::Registry.new('http://example.com') } + let(:repository) { registry.repository('group/test') } + let(:tag) { repository.tag('tag') } + let(:headers) { { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' } } + + it { expect(tag).to respond_to(:repository) } + it { expect(tag).to delegate_method(:registry).to(:repository) } + it { expect(tag).to delegate_method(:client).to(:repository) } + + context '#path' do + subject { tag.path } + + it { is_expected.to eq('example.com/group/test:tag') } + end + + context 'manifest processing' do + before do + stub_request(:get, 'http://example.com/v2/group/test/manifests/tag'). + with(headers: headers). + to_return( + status: 200, + body: File.read(Rails.root + 'spec/fixtures/container_registry/tag_manifest.json'), + headers: { 'Content-Type' => 'application/vnd.docker.distribution.manifest.v2+json' }) + end + + context '#layers' do + subject { tag.layers } + + it { expect(subject.length).to eq(1) } + end + + context '#total_size' do + subject { tag.total_size } + + it { is_expected.to eq(2319870) } + end + + context 'config processing' do + before do + stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac'). + with(headers: { 'Accept' => 'application/octet-stream' }). + to_return( + status: 200, + body: File.read(Rails.root + 'spec/fixtures/container_registry/config_blob.json')) + end + + context '#config' do + subject { tag.config } + + it { is_expected.to_not be_nil } + end + + context '#created_at' do + subject { tag.created_at } + + it { is_expected.to_not be_nil } + end + end + end + + context 'manifest digest' do + before do + stub_request(:head, 'http://example.com/v2/group/test/manifests/tag'). + with(headers: headers). + to_return(status: 200, headers: { 'Docker-Content-Digest' => 'sha256:digest' }) + end + + context '#digest' do + subject { tag.digest } + + it { is_expected.to eq('sha256:digest') } + end + + context '#delete' do + before do + stub_request(:delete, 'http://example.com/v2/group/test/manifests/sha256:digest'). + with(headers: headers). + to_return(status: 200) + end + + subject { tag.delete } + + it { is_expected.to be_truthy } + end + end +end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 4074f966299..4e68ac5e63a 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -70,6 +70,20 @@ describe Namespace, models: true do allow(@namespace).to receive(:path).and_return(new_path) expect(@namespace.move_dir).to be_truthy end + + context "when any project has container tags" do + before do + stub_container_registry_config(enabled: true) + stub_container_registry_tags('tag') + + create(:empty_project, namespace: @namespace) + + allow(@namespace).to receive(:path_was).and_return(@namespace.path) + allow(@namespace).to receive(:path).and_return('new_path') + end + + it { expect { @namespace.move_dir }.to raise_error('Namespace cannot be moved, because at least one project has tags in container registry') } + end end describe :rm_dir do diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index f6e5b132643..60e1ec43f2b 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -634,11 +634,11 @@ describe Project, models: true do # Project#gitlab_shell returns a new instance of Gitlab::Shell on every # call. This makes testing a bit easier. allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) - end - it 'renames a repository' do allow(project).to receive(:previous_changes).and_return('path' => ['foo']) + end + it 'renames a repository' do ns = project.namespace_dir expect(gitlab_shell).to receive(:mv_repository). @@ -663,6 +663,17 @@ describe Project, models: true do project.rename_repo end + + context 'container registry with tags' do + before do + stub_container_registry_config(enabled: true) + stub_container_registry_tags('tag') + end + + subject { project.rename_repo } + + it { expect{subject}.to raise_error(Exception) } + end end describe '#expire_caches_before_rename' do @@ -772,4 +783,71 @@ describe Project, models: true do expect(project.protected_branch?('foo')).to eq(false) end end + + describe '#container_registry_repository' do + let(:project) { create(:empty_project) } + + before { stub_container_registry_config(enabled: true) } + + subject { project.container_registry_repository } + + it { is_expected.to_not be_nil } + end + + describe '#container_registry_repository_url' do + let(:project) { create(:empty_project) } + + subject { project.container_registry_repository_url } + + before { stub_container_registry_config(**registry_settings) } + + context 'for enabled registry' do + let(:registry_settings) do + { + enabled: true, + host_port: 'example.com', + } + end + + it { is_expected.to_not be_nil } + end + + context 'for disabled registry' do + let(:registry_settings) do + { + enabled: false + } + end + + it { is_expected.to be_nil } + end + end + + describe '#has_container_registry_tags?' do + let(:project) { create(:empty_project) } + + subject { project.has_container_registry_tags? } + + context 'for enabled registry' do + before { stub_container_registry_config(enabled: true) } + + context 'with tags' do + before { stub_container_registry_tags('test', 'test2') } + + it { is_expected.to be_truthy } + end + + context 'when no tags' do + before { stub_container_registry_tags } + + it { is_expected.to be_falsey } + end + end + + context 'for disabled registry' do + before { stub_container_registry_config(enabled: false) } + + it { is_expected.to be_falsey } + end + end end diff --git a/spec/services/auth/container_registry_authentication_service_spec.rb b/spec/services/auth/container_registry_authentication_service_spec.rb index 3ea252ed44f..6c9f56a4fba 100644 --- a/spec/services/auth/container_registry_authentication_service_spec.rb +++ b/spec/services/auth/container_registry_authentication_service_spec.rb @@ -5,19 +5,12 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do let(:current_user) { nil } let(:current_params) { {} } let(:rsa_key) { OpenSSL::PKey::RSA.generate(512) } - let(:registry_settings) do - { - enabled: true, - issuer: 'rspec', - key: nil - } - end let(:payload) { JWT.decode(subject[:token], rsa_key).first } subject { described_class.new(current_project, current_user, current_params).execute } before do - allow(Gitlab.config.registry).to receive_messages(registry_settings) + stub_container_registry_config(enabled: true, issuer: 'rspec', key: nil) allow_any_instance_of(JSONWebToken::RSAToken).to receive(:key).and_return(rsa_key) end @@ -57,6 +50,11 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end end + shared_examples 'an unauthorized' do + it { is_expected.to include(http_status: 401) } + it { is_expected.to_not include(:token) } + end + shared_examples 'a forbidden' do it { is_expected.to include(http_status: 403) } it { is_expected.to_not include(:token) } @@ -123,7 +121,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do { offline_token: true } end - it_behaves_like 'a forbidden' + it_behaves_like 'an unauthorized' end context 'allow to pull and push images' do @@ -164,6 +162,20 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end end end + + context 'for project without container registry' do + let(:project) { create(:empty_project, :public, container_registry_enabled: false) } + + before { project.update(container_registry_enabled: false) } + + context 'disallow when pulling' do + let(:current_params) do + { scope: "repository:#{project.path_with_namespace}:pull" } + end + + it_behaves_like 'a forbidden' + end + end end context 'unauthorized' do @@ -172,7 +184,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do { offline_token: true } end - it_behaves_like 'a forbidden' + it_behaves_like 'an unauthorized' end context 'for invalid scope' do diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index a5cb6f382e4..29341c5e57e 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -28,6 +28,29 @@ describe Projects::DestroyService, services: true do it { expect(Dir.exist?(remove_path)).to be_truthy } end + context 'container registry' do + before do + stub_container_registry_config(enabled: true) + stub_container_registry_tags('tag') + end + + context 'tags deletion succeeds' do + it do + expect_any_instance_of(ContainerRegistry::Tag).to receive(:delete).and_return(true) + + destroy_project(project, user, {}) + end + end + + context 'tags deletion fails' do + before { expect_any_instance_of(ContainerRegistry::Tag).to receive(:delete).and_return(false) } + + subject { destroy_project(project, user, {}) } + + it { expect{subject}.to raise_error(Projects::DestroyService::DestroyError) } + end + end + def destroy_project(project, user, params) Projects::DestroyService.new(project, user, params).execute end diff --git a/spec/services/projects/transfer_service_spec.rb b/spec/services/projects/transfer_service_spec.rb index 06017317339..d5aa115a074 100644 --- a/spec/services/projects/transfer_service_spec.rb +++ b/spec/services/projects/transfer_service_spec.rb @@ -26,6 +26,17 @@ describe Projects::TransferService, services: true do it { expect(project.namespace).to eq(user.namespace) } end + context 'disallow transfering of project with tags' do + before do + stub_container_registry_config(enabled: true) + stub_container_registry_tags('tag') + end + + subject { transfer_project(project, user, group) } + + it { is_expected.to be_falsey } + end + context 'namespace -> not allowed namespace' do before do @result = transfer_project(project, user, group) diff --git a/spec/support/stub_gitlab_calls.rb b/spec/support/stub_gitlab_calls.rb index b5ca34bc028..f73416a3d0f 100644 --- a/spec/support/stub_gitlab_calls.rb +++ b/spec/support/stub_gitlab_calls.rb @@ -25,6 +25,23 @@ module StubGitlabCalls allow_any_instance_of(Project).to receive(:builds_enabled?).and_return(false) end + def stub_container_registry_config(registry_settings) + allow(Gitlab.config.registry).to receive_messages(registry_settings) + allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') + end + + def stub_container_registry_tags(*tags) + allow_any_instance_of(ContainerRegistry::Client).to receive(:repository_tags).and_return( + { "tags" => tags } + ) + allow_any_instance_of(ContainerRegistry::Client).to receive(:repository_manifest).and_return( + JSON.load(File.read(Rails.root + 'spec/fixtures/container_registry/tag_manifest.json')) + ) + allow_any_instance_of(ContainerRegistry::Client).to receive(:blob).and_return( + File.read(Rails.root + 'spec/fixtures/container_registry/config_blob.json') + ) + end + private def gitlab_url diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 05fc4c4554f..8aeb013eec6 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -21,7 +21,7 @@ describe 'gitlab:app namespace rake task' do end def reenable_backup_sub_tasks - %w{db repo uploads builds artifacts lfs}.each do |subtask| + %w{db repo uploads builds artifacts lfs registry}.each do |subtask| Rake::Task["gitlab:backup:#{subtask}:create"].reenable end end @@ -65,6 +65,7 @@ describe 'gitlab:app namespace rake task' do expect(Rake::Task['gitlab:backup:uploads:restore']).to receive(:invoke) expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive(:invoke) expect(Rake::Task['gitlab:backup:lfs:restore']).to receive(:invoke) + expect(Rake::Task['gitlab:backup:registry:restore']).to receive(:invoke) expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke) expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error end @@ -122,7 +123,7 @@ describe 'gitlab:app namespace rake task' do it 'should set correct permissions on the tar contents' do tar_contents, exit_status = Gitlab::Popen.popen( - %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz} + %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz registry.tar.gz} ) expect(exit_status).to eq(0) expect(tar_contents).to match('db/') @@ -131,12 +132,13 @@ describe 'gitlab:app namespace rake task' do expect(tar_contents).to match('builds.tar.gz') expect(tar_contents).to match('artifacts.tar.gz') expect(tar_contents).to match('lfs.tar.gz') - expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz)\/$/) + expect(tar_contents).to match('registry.tar.gz') + expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz|registry.tar.gz)\/$/) end it 'should delete temp directories' do temp_dirs = Dir.glob( - File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs}') + File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs,registry}') ) expect(temp_dirs).to be_empty @@ -172,7 +174,7 @@ describe 'gitlab:app namespace rake task' do it "does not contain skipped item" do tar_contents, _exit_status = Gitlab::Popen.popen( - %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz} + %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz registry.tar.gz} ) expect(tar_contents).to match('db/') @@ -180,6 +182,7 @@ describe 'gitlab:app namespace rake task' do expect(tar_contents).to match('builds.tar.gz') expect(tar_contents).to match('artifacts.tar.gz') expect(tar_contents).to match('lfs.tar.gz') + expect(tar_contents).to match('registry.tar.gz') expect(tar_contents).not_to match('repositories/') end @@ -195,6 +198,7 @@ describe 'gitlab:app namespace rake task' do expect(Rake::Task['gitlab:backup:builds:restore']).to receive :invoke expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive :invoke expect(Rake::Task['gitlab:backup:lfs:restore']).to receive :invoke + expect(Rake::Task['gitlab:backup:registry:restore']).to receive :invoke expect(Rake::Task['gitlab:shell:setup']).to receive :invoke expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error end |