Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42

13 files changed, 185 insertions, 26 deletions

diff --git a/lib/bulk_imports/clients/graphql.rb b/lib/bulk_imports/clients/graphql.rb
index ca549c4be14..0adc2b1c57f 100644
--- a/lib/bulk_imports/clients/graphql.rb
+++ b/lib/bulk_imports/clients/graphql.rb
@@ -23,15 +23,19 @@ module BulkImports
 
       attr_reader :client
 
-      delegate :query, :parse, :execute, to: :client
+      delegate :query, :parse, to: :client
 
       def initialize(url: Gitlab::Saas.com_url, token: nil)
         @url = Gitlab::Utils.append_path(url, '/api/graphql')
         @token = token
-        @client = Graphlient::Client.new(
-          @url,
-          options(http: HTTP)
-        )
+        @client = Graphlient::Client.new(@url, options(http: HTTP))
+        @compatible_instance_version = false
+      end
+
+      def execute(*args)
+        validate_instance_version!
+
+        client.execute(*args)
       end
 
       def options(extra = {})
@@ -44,6 +48,19 @@ module BulkImports
           }
         }.merge(extra)
       end
+
+      def validate_instance_version!
+        return if @compatible_instance_version
+
+        response = client.execute('{ metadata { version } }')
+        version = Gitlab::VersionInfo.parse(response.data.metadata.version)
+
+        if version.major < BulkImport::MINIMUM_GITLAB_MAJOR_VERSION
+          raise ::BulkImports::Error.unsupported_gitlab_version
+        else
+          @compatible_instance_version = true
+        end
+      end
     end
   end
 end
diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb
index c5f12d8c2ba..6c363a3552f 100644
--- a/lib/bulk_imports/clients/http.rb
+++ b/lib/bulk_imports/clients/http.rb
@@ -7,14 +7,13 @@ module BulkImports
       DEFAULT_PAGE = 1
       DEFAULT_PER_PAGE = 30
 
-      ConnectionError = Class.new(StandardError)
-
-      def initialize(uri:, token:, page: DEFAULT_PAGE, per_page: DEFAULT_PER_PAGE, api_version: API_VERSION)
-        @uri = URI.parse(uri)
+      def initialize(url:, token:, page: DEFAULT_PAGE, per_page: DEFAULT_PER_PAGE, api_version: API_VERSION)
+        @url = url
         @token = token&.strip
         @page = page
         @per_page = per_page
         @api_version = api_version
+        @compatible_instance_version = false
       end
 
       def get(resource, query = {})
@@ -53,10 +52,28 @@ module BulkImports
         Gitlab::Utils.append_path(api_url, resource)
       end
 
+      def validate_instance_version!
+        return if @compatible_instance_version
+
+        response = with_error_handling do
+          Gitlab::HTTP.get(resource_url(:version), default_options)
+        end
+
+        version = Gitlab::VersionInfo.parse(response.parsed_response['version'])
+
+        if version.major < BulkImport::MINIMUM_GITLAB_MAJOR_VERSION
+          raise ::BulkImports::Error.unsupported_gitlab_version
+        else
+          @compatible_instance_version = true
+        end
+      end
+
       private
 
       # rubocop:disable GitlabSecurity/PublicSend
       def request(method, resource, options = {}, &block)
+        validate_instance_version!
+
         with_error_handling do
           Gitlab::HTTP.public_send(
             method,
@@ -96,19 +113,15 @@ module BulkImports
       def with_error_handling
         response = yield
 
-        raise ConnectionError, "Error #{response.code}" unless response.success?
+        raise(::BulkImports::Error, "Error #{response.code}") unless response.success?
 
         response
       rescue *Gitlab::HTTP::HTTP_ERRORS => e
-        raise ConnectionError, e
-      end
-
-      def base_uri
-        @base_uri ||= "#{@uri.scheme}://#{@uri.host}:#{@uri.port}"
+        raise(::BulkImports::Error, e)
       end
 
       def api_url
-        Gitlab::Utils.append_path(base_uri, "/api/#{@api_version}")
+        Gitlab::Utils.append_path(@url, "/api/#{@api_version}")
       end
     end
   end
diff --git a/lib/bulk_imports/common/extractors/ndjson_extractor.rb b/lib/bulk_imports/common/extractors/ndjson_extractor.rb
index 79d626001a0..788d10ca364 100644
--- a/lib/bulk_imports/common/extractors/ndjson_extractor.rb
+++ b/lib/bulk_imports/common/extractors/ndjson_extractor.rb
@@ -7,6 +7,8 @@ module BulkImports
         include Gitlab::ImportExport::CommandLineUtil
         include Gitlab::Utils::StrongMemoize
 
+        FILE_SIZE_LIMIT = 5.gigabytes
+        ALLOWED_CONTENT_TYPES = %w(application/gzip application/octet-stream).freeze
         EXPORT_DOWNLOAD_URL_PATH = "/%{resource}/%{full_path}/export_relations/download?relation=%{relation}"
 
         def initialize(relation:)
@@ -39,7 +41,9 @@ module BulkImports
             configuration: context.configuration,
             relative_url: relative_resource_url(context),
             dir: tmp_dir,
-            filename: filename
+            filename: filename,
+            file_size_limit: FILE_SIZE_LIMIT,
+            allowed_content_types: ALLOWED_CONTENT_TYPES
           )
         end
 
diff --git a/lib/bulk_imports/common/extractors/rest_extractor.rb b/lib/bulk_imports/common/extractors/rest_extractor.rb
index 2179e0575c5..88315305222 100644
--- a/lib/bulk_imports/common/extractors/rest_extractor.rb
+++ b/lib/bulk_imports/common/extractors/rest_extractor.rb
@@ -25,7 +25,7 @@ module BulkImports
 
         def http_client(configuration)
           @http_client ||= BulkImports::Clients::HTTP.new(
-            uri: configuration.url,
+            url: configuration.url,
             token: configuration.access_token,
             per_page: 100
           )
diff --git a/lib/bulk_imports/error.rb b/lib/bulk_imports/error.rb
new file mode 100644
index 00000000000..0464aea642e
--- /dev/null
+++ b/lib/bulk_imports/error.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module BulkImports
+  class Error < StandardError
+    def self.unsupported_gitlab_version
+      self.new("Unsupported GitLab Version. Minimum Supported Gitlab Version #{BulkImport::MINIMUM_GITLAB_MAJOR_VERSION}.")
+    end
+  end
+end
diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb
index db5882d49a9..1140beef48c 100644
--- a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb
+++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb
@@ -18,7 +18,7 @@ module BulkImports
 
         def http_client(configuration)
           @http_client ||= BulkImports::Clients::HTTP.new(
-            uri: configuration.url,
+            url: configuration.url,
             token: configuration.access_token,
             per_page: 100
           )
diff --git a/lib/bulk_imports/groups/graphql/get_members_query.rb b/lib/bulk_imports/groups/graphql/get_members_query.rb
index e44d3c5aa9b..e76c87cc521 100644
--- a/lib/bulk_imports/groups/graphql/get_members_query.rb
+++ b/lib/bulk_imports/groups/graphql/get_members_query.rb
@@ -22,6 +22,7 @@ module BulkImports
                     integer_value: integerValue
                   }
                   user {
+                    user_gid: id
                     public_email: publicEmail
                   }
                 }
diff --git a/lib/bulk_imports/groups/pipelines/group_avatar_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_avatar_pipeline.rb
new file mode 100644
index 00000000000..6de8bbbc910
--- /dev/null
+++ b/lib/bulk_imports/groups/pipelines/group_avatar_pipeline.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Groups
+    module Pipelines
+      class GroupAvatarPipeline
+        include Pipeline
+
+        ALLOWED_AVATAR_DOWNLOAD_TYPES = (AvatarUploader::MIME_WHITELIST + %w(application/octet-stream)).freeze
+
+        GroupAvatarLoadingError = Class.new(StandardError)
+
+        def extract(context)
+          context.extra[:tmpdir] = Dir.mktmpdir
+
+          filepath = BulkImports::FileDownloadService.new(
+            configuration: context.configuration,
+            relative_url: "/groups/#{context.entity.encoded_source_full_path}/avatar",
+            dir: context.extra[:tmpdir],
+            file_size_limit: Avatarable::MAXIMUM_FILE_SIZE,
+            allowed_content_types: ALLOWED_AVATAR_DOWNLOAD_TYPES
+          ).execute
+
+          BulkImports::Pipeline::ExtractedData.new(data: { filepath: filepath })
+        end
+
+        def load(context, data)
+          return if data.blank?
+
+          File.open(data[:filepath]) do |avatar|
+            service = ::Groups::UpdateService.new(
+              portable,
+              current_user,
+              avatar: avatar
+            )
+
+            unless service.execute
+              raise GroupAvatarLoadingError, portable.errors.full_messages.first
+            end
+          end
+        end
+
+        def after_run(_)
+          FileUtils.remove_entry(context.extra[:tmpdir]) if context.extra[:tmpdir].present?
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/groups/pipelines/members_pipeline.rb b/lib/bulk_imports/groups/pipelines/members_pipeline.rb
index 5e4293d2c06..265abd5e3a7 100644
--- a/lib/bulk_imports/groups/pipelines/members_pipeline.rb
+++ b/lib/bulk_imports/groups/pipelines/members_pipeline.rb
@@ -15,6 +15,9 @@ module BulkImports
         def load(context, data)
           return unless data
 
+          # Current user is already a member
+          return if data['user_id'].to_i == context.current_user.id
+
           context.group.members.create!(data)
         end
       end
diff --git a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb
index e92c898171a..b9de375d0e9 100644
--- a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb
+++ b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb
@@ -6,18 +6,20 @@ module BulkImports
       class MemberAttributesTransformer
         def transform(context, data)
           data
-            .then { |data| add_user(data) }
+            .then { |data| add_user(data, context) }
             .then { |data| add_access_level(data) }
             .then { |data| add_author(data, context) }
         end
 
         private
 
-        def add_user(data)
+        def add_user(data, context)
           user = find_user(data&.dig('user', 'public_email'))
 
           return unless user
 
+          cache_source_user_id(data, user, context)
+
           data
             .except('user')
             .merge('user_id' => user.id)
@@ -48,6 +50,16 @@ module BulkImports
 
           data.merge('created_by_id' => context.current_user.id)
         end
+
+        def cache_source_user_id(data, user, context)
+          gid = data&.dig('user', 'user_gid')
+
+          return unless gid
+
+          source_user_id = GlobalID.parse(gid).model_id
+
+          ::BulkImports::UsersMapper.new(context: context).cache_source_user_id(source_user_id, user.id)
+        end
       end
     end
   end
diff --git a/lib/bulk_imports/ndjson_pipeline.rb b/lib/bulk_imports/ndjson_pipeline.rb
index 2de06bbcb88..93fd6986173 100644
--- a/lib/bulk_imports/ndjson_pipeline.rb
+++ b/lib/bulk_imports/ndjson_pipeline.rb
@@ -88,11 +88,7 @@ module BulkImports
       end
 
       def members_mapper
-        @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(
-          exported_members: [],
-          user: current_user,
-          importable: portable
-        )
+        @members_mapper ||= BulkImports::UsersMapper.new(context: context)
       end
     end
   end
diff --git a/lib/bulk_imports/stage.rb b/lib/bulk_imports/stage.rb
index bc7fc14b5a0..b1bceecbaea 100644
--- a/lib/bulk_imports/stage.rb
+++ b/lib/bulk_imports/stage.rb
@@ -9,6 +9,10 @@ module BulkImports
         pipeline: BulkImports::Groups::Pipelines::GroupPipeline,
         stage: 0
       },
+      avatar: {
+        pipeline: BulkImports::Groups::Pipelines::GroupAvatarPipeline,
+        stage: 1
+      },
       subgroups: {
         pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline,
         stage: 1
diff --git a/lib/bulk_imports/users_mapper.rb b/lib/bulk_imports/users_mapper.rb
new file mode 100644
index 00000000000..74412bc3831
--- /dev/null
+++ b/lib/bulk_imports/users_mapper.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module BulkImports
+  class UsersMapper
+    include Gitlab::Utils::StrongMemoize
+
+    SOURCE_USER_IDS_CACHE_KEY = 'bulk_imports/%{bulk_import}/%{entity}/source_user_ids'
+
+    def initialize(context:)
+      @context = context
+      @cache_key = SOURCE_USER_IDS_CACHE_KEY % {
+        bulk_import: @context.bulk_import.id,
+        entity: @context.entity.id
+      }
+    end
+
+    def map
+      strong_memoize(:map) do
+        map = hash_with_default
+
+        cached_source_user_ids.each_pair do |source_id, destination_id|
+          map[source_id.to_i] = destination_id.to_i
+        end
+
+        map
+      end
+    end
+
+    def include?(source_user_id)
+      map.has_key?(source_user_id)
+    end
+
+    def default_user_id
+      @context.current_user.id
+    end
+
+    def cache_source_user_id(source_id, destination_id)
+      ::Gitlab::Cache::Import::Caching.hash_add(@cache_key, source_id, destination_id)
+    end
+
+    private
+
+    def hash_with_default
+      Hash.new { default_user_id }
+    end
+
+    def cached_source_user_ids
+      ::Gitlab::Cache::Import::Caching.values_from_hash(@cache_key)
+    end
+  end
+end