5 files changed, 288 insertions, 130 deletions

diff --git a/lib/container_registry/base_client.rb b/lib/container_registry/base_client.rb
new file mode 100644
index 00000000000..22d4510fe71
--- /dev/null
+++ b/lib/container_registry/base_client.rb
@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'faraday_middleware'
+require 'digest'
+
+module ContainerRegistry
+  class BaseClient
+    DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json'
+    DOCKER_DISTRIBUTION_MANIFEST_LIST_V2_TYPE = 'application/vnd.docker.distribution.manifest.list.v2+json'
+    OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json'
+    CONTAINER_IMAGE_V1_TYPE = 'application/vnd.docker.container.image.v1+json'
+
+    ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze
+    ACCEPTED_TYPES_RAW = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE, DOCKER_DISTRIBUTION_MANIFEST_LIST_V2_TYPE].freeze
+
+    RETRY_EXCEPTIONS = [Faraday::Request::Retry::DEFAULT_EXCEPTIONS, Faraday::ConnectionFailed].flatten.freeze
+    RETRY_OPTIONS = {
+      max: 1,
+      interval: 5,
+      exceptions: RETRY_EXCEPTIONS
+    }.freeze
+
+    ERROR_CALLBACK_OPTIONS = {
+      callback: -> (env, exception) do
+        Gitlab::ErrorTracking.log_exception(
+          exception,
+          class: name,
+          url: env[:url]
+        )
+      end
+    }.freeze
+
+    # Taken from: FaradayMiddleware::FollowRedirects
+    REDIRECT_CODES = Set.new [301, 302, 303, 307]
+
+    class << self
+      private
+
+      def with_dummy_client(return_value_if_disabled: nil)
+        registry_config = Gitlab.config.registry
+        unless registry_config.enabled && registry_config.api_url.present?
+          return return_value_if_disabled
+        end
+
+        token = Auth::ContainerRegistryAuthenticationService.access_token([], [])
+        yield new(registry_config.api_url, token: token)
+      end
+    end
+
+    def initialize(base_uri, options = {})
+      @base_uri = base_uri
+      @options = options
+    end
+
+    private
+
+    def faraday(timeout_enabled: true)
+      @faraday ||= faraday_base(timeout_enabled: timeout_enabled) do |conn|
+        initialize_connection(conn, @options, &method(:configure_connection))
+      end
+    end
+
+    def faraday_base(timeout_enabled: true, &block)
+      request_options = timeout_enabled ? Gitlab::HTTP::DEFAULT_TIMEOUT_OPTIONS : nil
+
+      Faraday.new(
+        @base_uri,
+        headers: { user_agent: "GitLab/#{Gitlab::VERSION}" },
+        request: request_options,
+        &block
+      )
+    end
+
+    def initialize_connection(conn, options)
+      conn.request :json
+
+      if options[:user] && options[:password]
+        conn.request(:basic_auth, options[:user].to_s, options[:password].to_s)
+      elsif options[:token]
+        conn.request(:authorization, :bearer, options[:token].to_s)
+      end
+
+      yield(conn) if block_given?
+
+      conn.request(:retry, RETRY_OPTIONS)
+      conn.request(:gitlab_error_callback, ERROR_CALLBACK_OPTIONS)
+      conn.adapter :net_http
+    end
+
+    def response_body(response, allow_redirect: false)
+      if allow_redirect && REDIRECT_CODES.include?(response.status)
+        response = redirect_response(response.headers['location'])
+      end
+
+      response.body if response && response.success?
+    end
+
+    def redirect_response(location)
+      return unless location
+
+      uri = URI(@base_uri).merge(location)
+      raise ArgumentError, "Invalid scheme for #{location}" unless %w[http https].include?(uri.scheme)
+
+      faraday_redirect.get(uri)
+    end
+
+    def configure_connection(conn)
+      conn.headers['Accept'] = ACCEPTED_TYPES
+
+      conn.response :json, content_type: 'application/json'
+      conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws'
+      conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json'
+      conn.response :json, content_type: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE
+      conn.response :json, content_type: OCI_MANIFEST_V1_TYPE
+    end
+
+    # Create a new request to make sure the Authorization header is not inserted
+    # via the Faraday middleware
+    def faraday_redirect
+      @faraday_redirect ||= faraday_base do |conn|
+        conn.request :json
+
+        conn.request(:retry, RETRY_OPTIONS)
+        conn.request(:gitlab_error_callback, ERROR_CALLBACK_OPTIONS)
+        conn.adapter :net_http
+      end
+    end
+
+    def delete_if_exists(path)
+      result = faraday.delete(path)
+
+      result.success? || result.status == 404
+    end
+  end
+end
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
index c2ad9e6ae89..add238350dd 100644
--- a/lib/container_registry/client.rb
+++ b/lib/container_registry/client.rb
@@ -1,68 +1,25 @@
 # frozen_string_literal: true
 
-require 'faraday'
-require 'faraday_middleware'
-require 'digest'
-
 module ContainerRegistry
-  class Client
+  class Client < BaseClient
     include Gitlab::Utils::StrongMemoize
 
     attr_accessor :uri
 
-    DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json'
-    DOCKER_DISTRIBUTION_MANIFEST_LIST_V2_TYPE = 'application/vnd.docker.distribution.manifest.list.v2+json'
-    OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json'
-    CONTAINER_IMAGE_V1_TYPE = 'application/vnd.docker.container.image.v1+json'
     REGISTRY_VERSION_HEADER = 'gitlab-container-registry-version'
     REGISTRY_FEATURES_HEADER = 'gitlab-container-registry-features'
     REGISTRY_TAG_DELETE_FEATURE = 'tag_delete'
 
-    ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze
-
-    ACCEPTED_TYPES_RAW = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE, DOCKER_DISTRIBUTION_MANIFEST_LIST_V2_TYPE].freeze
-
-    # Taken from: FaradayMiddleware::FollowRedirects
-    REDIRECT_CODES = Set.new [301, 302, 303, 307]
-
-    RETRY_EXCEPTIONS = [Faraday::Request::Retry::DEFAULT_EXCEPTIONS, Faraday::ConnectionFailed].flatten.freeze
-    RETRY_OPTIONS = {
-      max: 1,
-      interval: 5,
-      exceptions: RETRY_EXCEPTIONS
-    }.freeze
-
-    ERROR_CALLBACK_OPTIONS = {
-      callback: -> (env, exception) do
-        Gitlab::ErrorTracking.log_exception(
-          exception,
-          class: name,
-          url: env[:url]
-        )
-      end
-    }.freeze
-
     def self.supports_tag_delete?
-      registry_config = Gitlab.config.registry
-      return false unless registry_config.enabled && registry_config.api_url.present?
-
-      token = Auth::ContainerRegistryAuthenticationService.access_token([], [])
-      client = new(registry_config.api_url, token: token)
-      client.supports_tag_delete?
+      with_dummy_client(return_value_if_disabled: false) do |client|
+        client.supports_tag_delete?
+      end
     end
 
     def self.registry_info
-      registry_config = Gitlab.config.registry
-      return unless registry_config.enabled && registry_config.api_url.present?
-
-      token = Auth::ContainerRegistryAuthenticationService.access_token([], [])
-      client = new(registry_config.api_url, token: token)
-      client.registry_info
-    end
-
-    def initialize(base_uri, options = {})
-      @base_uri = base_uri
-      @options = options
+      with_dummy_client do |client|
+        client.registry_info
+      end
     end
 
     def registry_info
@@ -176,89 +133,11 @@ module ContainerRegistry
 
     private
 
-    def initialize_connection(conn, options)
-      conn.request :json
-
-      if options[:user] && options[:password]
-        conn.request(:basic_auth, options[:user].to_s, options[:password].to_s)
-      elsif options[:token]
-        conn.request(:authorization, :bearer, options[:token].to_s)
-      end
-
-      yield(conn) if block_given?
-
-      conn.request(:retry, RETRY_OPTIONS)
-      conn.request(:gitlab_error_callback, ERROR_CALLBACK_OPTIONS)
-      conn.adapter :net_http
-    end
-
-    def accept_manifest(conn)
-      conn.headers['Accept'] = ACCEPTED_TYPES
-
-      conn.response :json, content_type: 'application/json'
-      conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws'
-      conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json'
-      conn.response :json, content_type: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE
-      conn.response :json, content_type: OCI_MANIFEST_V1_TYPE
-    end
-
-    def response_body(response, allow_redirect: false)
-      if allow_redirect && REDIRECT_CODES.include?(response.status)
-        response = redirect_response(response.headers['location'])
-      end
-
-      response.body if response && response.success?
-    end
-
-    def redirect_response(location)
-      return unless location
-
-      uri = URI(@base_uri).merge(location)
-      raise ArgumentError, "Invalid scheme for #{location}" unless %w[http https].include?(uri.scheme)
-
-      faraday_redirect.get(uri)
-    end
-
-    def faraday(timeout_enabled: true)
-      @faraday ||= faraday_base(timeout_enabled: timeout_enabled) do |conn|
-        initialize_connection(conn, @options, &method(:accept_manifest))
-      end
-    end
-
     def faraday_blob
       @faraday_blob ||= faraday_base do |conn|
         initialize_connection(conn, @options)
       end
     end
-
-    # Create a new request to make sure the Authorization header is not inserted
-    # via the Faraday middleware
-    def faraday_redirect
-      @faraday_redirect ||= faraday_base do |conn|
-        conn.request :json
-
-        conn.request(:retry, RETRY_OPTIONS)
-        conn.request(:gitlab_error_callback, ERROR_CALLBACK_OPTIONS)
-        conn.adapter :net_http
-      end
-    end
-
-    def faraday_base(timeout_enabled: true, &block)
-      request_options = timeout_enabled ? Gitlab::HTTP::DEFAULT_TIMEOUT_OPTIONS : nil
-
-      Faraday.new(
-        @base_uri,
-        headers: { user_agent: "GitLab/#{Gitlab::VERSION}" },
-        request: request_options,
-        &block
-      )
-    end
-
-    def delete_if_exists(path)
-      result = faraday.delete(path)
-
-      result.success? || result.status == 404
-    end
   end
 end
 
diff --git a/lib/container_registry/gitlab_api_client.rb b/lib/container_registry/gitlab_api_client.rb
new file mode 100644
index 00000000000..20b8e1d419b
--- /dev/null
+++ b/lib/container_registry/gitlab_api_client.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module ContainerRegistry
+  class GitlabApiClient < BaseClient
+    include Gitlab::Utils::StrongMemoize
+
+    JSON_TYPE = 'application/json'
+
+    IMPORT_RESPONSES = {
+      200 => :already_imported,
+      202 => :ok,
+      401 => :unauthorized,
+      404 => :not_found,
+      409 => :already_being_imported,
+      424 => :pre_import_failed,
+      425 => :already_being_imported,
+      429 => :too_many_imports
+    }.freeze
+
+    REGISTRY_GITLAB_V1_API_FEATURE = 'gitlab_v1_api'
+
+    def self.supports_gitlab_api?
+      with_dummy_client(return_value_if_disabled: false) do |client|
+        client.supports_gitlab_api?
+      end
+    end
+
+    # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#compliance-check
+    def supports_gitlab_api?
+      strong_memoize(:supports_gitlab_api) do
+        registry_features = Gitlab::CurrentSettings.container_registry_features || []
+        next true if ::Gitlab.com? && registry_features.include?(REGISTRY_GITLAB_V1_API_FEATURE)
+
+        response = faraday.get('/gitlab/v1/')
+        response.success? || response.status == 401
+      end
+    end
+
+    # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#import-repository
+    def pre_import_repository(path)
+      response = start_import_for(path, pre: true)
+      IMPORT_RESPONSES.fetch(response.status, :error)
+    end
+
+    # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#import-repository
+    def import_repository(path)
+      response = start_import_for(path, pre: false)
+      IMPORT_RESPONSES.fetch(response.status, :error)
+    end
+
+    # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#get-repository-import-status
+    def import_status(path)
+      body_hash = response_body(faraday.get(import_url_for(path)))
+      body_hash['status'] || 'error'
+    end
+
+    private
+
+    def start_import_for(path, pre:)
+      faraday.put(import_url_for(path)) do |req|
+        req.params['pre'] = pre.to_s
+      end
+    end
+
+    def import_url_for(path)
+      "/gitlab/v1/import/#{path}/"
+    end
+
+    # overrides the default configuration
+    def configure_connection(conn)
+      conn.headers['Accept'] = [JSON_TYPE]
+
+      conn.response :json, content_type: JSON_TYPE
+    end
+  end
+end
diff --git a/lib/container_registry/migration.rb b/lib/container_registry/migration.rb
new file mode 100644
index 00000000000..b03c94e5ebf
--- /dev/null
+++ b/lib/container_registry/migration.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module ContainerRegistry
+  module Migration
+    class << self
+      delegate :container_registry_import_max_tags_count, to: ::Gitlab::CurrentSettings
+      delegate :container_registry_import_max_retries, to: ::Gitlab::CurrentSettings
+      delegate :container_registry_import_start_max_retries, to: ::Gitlab::CurrentSettings
+      delegate :container_registry_import_max_step_duration, to: ::Gitlab::CurrentSettings
+      delegate :container_registry_import_target_plan, to: ::Gitlab::CurrentSettings
+      delegate :container_registry_import_created_before, to: ::Gitlab::CurrentSettings
+
+      alias_method :max_tags_count, :container_registry_import_max_tags_count
+      alias_method :max_retries, :container_registry_import_max_retries
+      alias_method :start_max_retries, :container_registry_import_start_max_retries
+      alias_method :max_step_duration, :container_registry_import_max_step_duration
+      alias_method :target_plan_name, :container_registry_import_target_plan
+      alias_method :created_before, :container_registry_import_created_before
+    end
+
+    def self.enabled?
+      Feature.enabled?(:container_registry_migration_phase2_enabled)
+    end
+
+    def self.limit_gitlab_org?
+      Feature.enabled?(:container_registry_migration_limit_gitlab_org)
+    end
+
+    def self.enqueue_waiting_time
+      return 0 if Feature.enabled?(:container_registry_migration_phase2_enqueue_speed_fast)
+      return 6.hours if Feature.enabled?(:container_registry_migration_phase2_enqueue_speed_slow)
+
+      1.hour
+    end
+
+    def self.capacity
+      # Increasing capacity numbers will increase the n+1 API calls we can have
+      # in ContainerRegistry::Migration::GuardWorker#external_migration_in_progress?
+      #
+      # TODO: See https://gitlab.com/gitlab-org/container-registry/-/issues/582
+      #
+      return 25 if Feature.enabled?(:container_registry_migration_phase2_capacity_25)
+      return 10 if Feature.enabled?(:container_registry_migration_phase2_capacity_10)
+      return 1 if Feature.enabled?(:container_registry_migration_phase2_capacity_1)
+
+      0
+    end
+
+    def self.target_plan
+      Plan.find_by_name(target_plan_name)
+    end
+  end
+end
diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb
index 523364ac7c7..710f8169a00 100644
--- a/lib/container_registry/registry.rb
+++ b/lib/container_registry/registry.rb
@@ -2,12 +2,26 @@
 
 module ContainerRegistry
   class Registry
+    include Gitlab::Utils::StrongMemoize
+
     attr_reader :uri, :client, :path
 
     def initialize(uri, options = {})
       @uri = uri
-      @path = options[:path] || default_path
-      @client = ContainerRegistry::Client.new(uri, options)
+      @options = options
+      @path = @options[:path] || default_path
+      @client = ContainerRegistry::Client.new(@uri, @options)
+    end
+
+    def gitlab_api_client
+      strong_memoize(:gitlab_api_client) do
+        token = Auth::ContainerRegistryAuthenticationService.import_access_token
+
+        url = Gitlab.config.registry.api_url
+        host_port = Gitlab.config.registry.host_port
+
+        ContainerRegistry::GitlabApiClient.new(url, token: token, path: host_port)
+      end
     end
 
     private