Backport of https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3809ce-docker_image_replication

Introducing Docker Registry replication

9 files changed, 57 insertions, 3 deletions

diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb
index facd81dde80..2a5ae7930e6 100644
--- a/app/models/container_repository.rb
+++ b/app/models/container_repository.rb
@@ -70,10 +70,14 @@ class ContainerRepository < ApplicationRecord
     digests = tags.map { |tag| tag.digest }.to_set
 
     digests.all? do |digest|
-      client.delete_repository_tag(self.path, digest)
+      delete_tag_by_digest(digest)
     end
   end
 
+  def delete_tag_by_digest(digest)
+    client.delete_repository_tag(self.path, digest)
+  end
+
   def self.build_from_path(path)
     self.new(project: path.repository_project,
              name: path.repository_name)
diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb
index 707caee482c..0a069320936 100644
--- a/app/services/auth/container_registry_authentication_service.rb
+++ b/app/services/auth/container_registry_authentication_service.rb
@@ -17,6 +17,14 @@ module Auth
     end
 
     def self.full_access_token(*names)
+      access_token(%w(*), names)
+    end
+
+    def self.pull_access_token(*names)
+      access_token(['pull'], names)
+    end
+
+    def self.access_token(actions, names)
       names = names.flatten
       registry = Gitlab.config.registry
       token = JSONWebToken::RSAToken.new(registry.key)
@@ -25,7 +33,7 @@ module Auth
       token.expire_time = token_expire_at
 
       token[:access] = names.map do |name|
-        { type: 'repository', name: name, actions: %w(*) }
+        { type: 'repository', name: name, actions: actions }
       end
 
       token.encoded
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index dd53127ac2c..39b719a5978 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -427,6 +427,11 @@ production: &base
     # If it is blank, it defaults to external_url.
     node_name: ''
 
+    registry_replication:
+      # enabled: true
+      # primary_api_url: http://localhost:5000/ # internal address to the primary registry, will be used by GitLab to directly communicate with primary registry API
+
+
   #
   # 2. GitLab CI settings
   # ==========================
diff --git a/config/initializers/0_inflections.rb b/config/initializers/0_inflections.rb
index 4d1f4917275..d317825c1b8 100644
--- a/config/initializers/0_inflections.rb
+++ b/config/initializers/0_inflections.rb
@@ -19,6 +19,7 @@ ActiveSupport::Inflector.inflections do |inflect|
     project_registry
     file_registry
     job_artifact_registry
+    container_repository_registry
     vulnerability_feedback
     vulnerabilities_feedback
     group_view
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 32fec7c3d22..659801f787d 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -296,6 +296,12 @@ Gitlab.ee do
   Settings['geo'] ||= Settingslogic.new({})
   # For backwards compatibility, default to gitlab_url and if so, ensure it ends with "/"
   Settings.geo['node_name'] = Settings.geo['node_name'].presence || Settings.gitlab['url'].chomp('/').concat('/')
+
+  #
+  # Registry replication
+  #
+  Settings.geo['registry_replication'] ||= Settingslogic.new({})
+  Settings.geo.registry_replication['enabled'] ||= false
 end
 
 #
@@ -473,6 +479,9 @@ Gitlab.ee do
   Settings.cron_jobs['geo_repository_verification_secondary_scheduler_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['geo_repository_verification_secondary_scheduler_worker']['cron'] ||= '*/1 * * * *'
   Settings.cron_jobs['geo_repository_verification_secondary_scheduler_worker']['job_class'] ||= 'Geo::RepositoryVerification::Secondary::SchedulerWorker'
+  Settings.cron_jobs['geo_container_repository_sync_worker'] ||= Settingslogic.new({})
+  Settings.cron_jobs['geo_container_repository_sync_worker']['cron'] ||= '*/1 * * * *'
+  Settings.cron_jobs['geo_container_repository_sync_worker']['job_class'] ||= 'Geo::ContainerRepositorySyncDispatchWorker'
   Settings.cron_jobs['historical_data_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['historical_data_worker']['cron'] ||= '0 12 * * *'
   Settings.cron_jobs['historical_data_worker']['job_class'] = 'HistoricalDataWorker'
diff --git a/db/migrate/20190612111404_add_geo_container_sync_capacity.rb b/db/migrate/20190612111404_add_geo_container_sync_capacity.rb
new file mode 100644
index 00000000000..d4cd569f460
--- /dev/null
+++ b/db/migrate/20190612111404_add_geo_container_sync_capacity.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class AddGeoContainerSyncCapacity < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    change_table :geo_nodes do |t|
+      t.column :container_repositories_max_capacity, :integer, default: 10, null: false
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 6f5fc6c65eb..804f77b91de 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -1435,6 +1435,7 @@ ActiveRecord::Schema.define(version: 2019_07_29_090456) do
     t.integer "minimum_reverification_interval", default: 7, null: false
     t.string "internal_url"
     t.string "name", null: false
+    t.integer "container_repositories_max_capacity", default: 10, null: false
     t.index ["access_key"], name: "index_geo_nodes_on_access_key"
     t.index ["name"], name: "index_geo_nodes_on_name", unique: true
     t.index ["primary"], name: "index_geo_nodes_on_primary"
diff --git a/spec/factories/container_repositories.rb b/spec/factories/container_repositories.rb
index a9771200d6e..0b756220d68 100644
--- a/spec/factories/container_repositories.rb
+++ b/spec/factories/container_repositories.rb
@@ -2,7 +2,7 @@
 
 FactoryBot.define do
   factory :container_repository do
-    name 'test_image'
+    sequence(:name) { |n| "test_image_#{n}" }
     project
 
     transient do
diff --git a/spec/services/auth/container_registry_authentication_service_spec.rb b/spec/services/auth/container_registry_authentication_service_spec.rb
index 4f4776bbb27..3ca389ba25b 100644
--- a/spec/services/auth/container_registry_authentication_service_spec.rb
+++ b/spec/services/auth/container_registry_authentication_service_spec.rb
@@ -145,6 +145,19 @@ describe Auth::ContainerRegistryAuthenticationService do
     it_behaves_like 'not a container repository factory'
   end
 
+  describe '#pull_access_token' do
+    let(:project) { create(:project) }
+    let(:token) { described_class.pull_access_token(project.full_path) }
+
+    subject { { token: token } }
+
+    it_behaves_like 'an accessible' do
+      let(:actions) { ['pull'] }
+    end
+
+    it_behaves_like 'not a container repository factory'
+  end
+
   context 'user authorization' do
     let(:current_user) { create(:user) }