From f519a4b72f81a1e3c81e5e684d236bbe30e0dd2d Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Thu, 1 Aug 2019 15:03:08 +0300 Subject: Backport of https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3809 Introducing Docker Registry replication --- app/models/container_repository.rb | 6 +++++- .../auth/container_registry_authentication_service.rb | 10 +++++++++- config/gitlab.yml.example | 5 +++++ config/initializers/0_inflections.rb | 1 + config/initializers/1_settings.rb | 9 +++++++++ .../20190612111404_add_geo_container_sync_capacity.rb | 13 +++++++++++++ db/schema.rb | 1 + spec/factories/container_repositories.rb | 2 +- .../auth/container_registry_authentication_service_spec.rb | 13 +++++++++++++ 9 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 db/migrate/20190612111404_add_geo_container_sync_capacity.rb diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index facd81dde80..2a5ae7930e6 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -70,10 +70,14 @@ class ContainerRepository < ApplicationRecord digests = tags.map { |tag| tag.digest }.to_set digests.all? do |digest| - client.delete_repository_tag(self.path, digest) + delete_tag_by_digest(digest) end end + def delete_tag_by_digest(digest) + client.delete_repository_tag(self.path, digest) + end + def self.build_from_path(path) self.new(project: path.repository_project, name: path.repository_name) diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 707caee482c..0a069320936 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -17,6 +17,14 @@ module Auth end def self.full_access_token(*names) + access_token(%w(*), names) + end + + def self.pull_access_token(*names) + access_token(['pull'], names) + end + + def self.access_token(actions, names) names = names.flatten registry = Gitlab.config.registry token = JSONWebToken::RSAToken.new(registry.key) @@ -25,7 +33,7 @@ module Auth token.expire_time = token_expire_at token[:access] = names.map do |name| - { type: 'repository', name: name, actions: %w(*) } + { type: 'repository', name: name, actions: actions } end token.encoded diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index dd53127ac2c..39b719a5978 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -427,6 +427,11 @@ production: &base # If it is blank, it defaults to external_url. node_name: '' + registry_replication: + # enabled: true + # primary_api_url: http://localhost:5000/ # internal address to the primary registry, will be used by GitLab to directly communicate with primary registry API + + # # 2. GitLab CI settings # ========================== diff --git a/config/initializers/0_inflections.rb b/config/initializers/0_inflections.rb index 4d1f4917275..d317825c1b8 100644 --- a/config/initializers/0_inflections.rb +++ b/config/initializers/0_inflections.rb @@ -19,6 +19,7 @@ ActiveSupport::Inflector.inflections do |inflect| project_registry file_registry job_artifact_registry + container_repository_registry vulnerability_feedback vulnerabilities_feedback group_view diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 32fec7c3d22..659801f787d 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -296,6 +296,12 @@ Gitlab.ee do Settings['geo'] ||= Settingslogic.new({}) # For backwards compatibility, default to gitlab_url and if so, ensure it ends with "/" Settings.geo['node_name'] = Settings.geo['node_name'].presence || Settings.gitlab['url'].chomp('/').concat('/') + + # + # Registry replication + # + Settings.geo['registry_replication'] ||= Settingslogic.new({}) + Settings.geo.registry_replication['enabled'] ||= false end # @@ -473,6 +479,9 @@ Gitlab.ee do Settings.cron_jobs['geo_repository_verification_secondary_scheduler_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['geo_repository_verification_secondary_scheduler_worker']['cron'] ||= '*/1 * * * *' Settings.cron_jobs['geo_repository_verification_secondary_scheduler_worker']['job_class'] ||= 'Geo::RepositoryVerification::Secondary::SchedulerWorker' + Settings.cron_jobs['geo_container_repository_sync_worker'] ||= Settingslogic.new({}) + Settings.cron_jobs['geo_container_repository_sync_worker']['cron'] ||= '*/1 * * * *' + Settings.cron_jobs['geo_container_repository_sync_worker']['job_class'] ||= 'Geo::ContainerRepositorySyncDispatchWorker' Settings.cron_jobs['historical_data_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['historical_data_worker']['cron'] ||= '0 12 * * *' Settings.cron_jobs['historical_data_worker']['job_class'] = 'HistoricalDataWorker' diff --git a/db/migrate/20190612111404_add_geo_container_sync_capacity.rb b/db/migrate/20190612111404_add_geo_container_sync_capacity.rb new file mode 100644 index 00000000000..d4cd569f460 --- /dev/null +++ b/db/migrate/20190612111404_add_geo_container_sync_capacity.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddGeoContainerSyncCapacity < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + change_table :geo_nodes do |t| + t.column :container_repositories_max_capacity, :integer, default: 10, null: false + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 6f5fc6c65eb..804f77b91de 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1435,6 +1435,7 @@ ActiveRecord::Schema.define(version: 2019_07_29_090456) do t.integer "minimum_reverification_interval", default: 7, null: false t.string "internal_url" t.string "name", null: false + t.integer "container_repositories_max_capacity", default: 10, null: false t.index ["access_key"], name: "index_geo_nodes_on_access_key" t.index ["name"], name: "index_geo_nodes_on_name", unique: true t.index ["primary"], name: "index_geo_nodes_on_primary" diff --git a/spec/factories/container_repositories.rb b/spec/factories/container_repositories.rb index a9771200d6e..0b756220d68 100644 --- a/spec/factories/container_repositories.rb +++ b/spec/factories/container_repositories.rb @@ -2,7 +2,7 @@ FactoryBot.define do factory :container_repository do - name 'test_image' + sequence(:name) { |n| "test_image_#{n}" } project transient do diff --git a/spec/services/auth/container_registry_authentication_service_spec.rb b/spec/services/auth/container_registry_authentication_service_spec.rb index 4f4776bbb27..3ca389ba25b 100644 --- a/spec/services/auth/container_registry_authentication_service_spec.rb +++ b/spec/services/auth/container_registry_authentication_service_spec.rb @@ -145,6 +145,19 @@ describe Auth::ContainerRegistryAuthenticationService do it_behaves_like 'not a container repository factory' end + describe '#pull_access_token' do + let(:project) { create(:project) } + let(:token) { described_class.pull_access_token(project.full_path) } + + subject { { token: token } } + + it_behaves_like 'an accessible' do + let(:actions) { ['pull'] } + end + + it_behaves_like 'not a container repository factory' + end + context 'user authorization' do let(:current_user) { create(:user) } -- cgit v1.2.1