Add latest changes from gitlab-org/gitlab@15-0-stable-eev15.0.0-rc42

12 files changed, 176 insertions, 21 deletions

diff --git a/lib/bulk_imports/common/extractors/json_extractor.rb b/lib/bulk_imports/common/extractors/json_extractor.rb
new file mode 100644
index 00000000000..45a48cedce8
--- /dev/null
+++ b/lib/bulk_imports/common/extractors/json_extractor.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Common
+    module Extractors
+      class JsonExtractor
+        def initialize(relation:)
+          @relation = relation
+          @tmpdir = Dir.mktmpdir
+        end
+
+        def extract(context)
+          download_service(context).execute
+          decompression_service.execute
+
+          attributes = ndjson_reader.consume_attributes(relation)
+
+          BulkImports::Pipeline::ExtractedData.new(data: attributes)
+        end
+
+        def remove_tmpdir
+          FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir)
+        end
+
+        private
+
+        attr_reader :relation, :tmpdir
+
+        def filename
+          "#{relation}.json.gz"
+        end
+
+        def download_service(context)
+          @download_service ||= BulkImports::FileDownloadService.new(
+            configuration: context.configuration,
+            relative_url: context.entity.relation_download_url_path(relation),
+            tmpdir: tmpdir,
+            filename: filename
+          )
+        end
+
+        def decompression_service
+          @decompression_service ||= BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: filename)
+        end
+
+        def ndjson_reader
+          @ndjson_reader ||= Gitlab::ImportExport::Json::NdjsonReader.new(tmpdir)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/common/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb
index 0f4def3b17a..915dcf1b455 100644
--- a/lib/bulk_imports/common/pipelines/entity_finisher.rb
+++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb
@@ -4,7 +4,7 @@ module BulkImports
   module Common
     module Pipelines
       class EntityFinisher
-        def self.ndjson_pipeline?
+        def self.file_extraction_pipeline?
           false
         end
 
diff --git a/lib/bulk_imports/groups/pipelines/group_attributes_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_attributes_pipeline.rb
new file mode 100644
index 00000000000..d4f5901e971
--- /dev/null
+++ b/lib/bulk_imports/groups/pipelines/group_attributes_pipeline.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Groups
+    module Pipelines
+      class GroupAttributesPipeline
+        include Pipeline
+
+        file_extraction_pipeline!
+
+        relation_name BulkImports::FileTransfer::BaseConfig::SELF_RELATION
+
+        extractor ::BulkImports::Common::Extractors::JsonExtractor, relation: relation
+
+        transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer
+
+        def transform(_context, data)
+          return unless data
+
+          data.symbolize_keys.slice(:membership_lock)
+        end
+
+        def load(_context, data)
+          return unless data
+
+          ::Groups::UpdateService.new(portable, current_user, data).execute
+        end
+
+        def after_run(_context)
+          extractor.remove_tmpdir
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/groups/pipelines/namespace_settings_pipeline.rb b/lib/bulk_imports/groups/pipelines/namespace_settings_pipeline.rb
new file mode 100644
index 00000000000..1bd6486b413
--- /dev/null
+++ b/lib/bulk_imports/groups/pipelines/namespace_settings_pipeline.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Groups
+    module Pipelines
+      class NamespaceSettingsPipeline
+        include Pipeline
+
+        file_extraction_pipeline!
+
+        relation_name 'namespace_settings'
+
+        extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation
+
+        transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer
+
+        def transform(_context, data)
+          return unless data
+
+          data.first.symbolize_keys.slice(*allowed_attributes)
+        end
+
+        def load(_context, data)
+          return unless data
+
+          ::Groups::UpdateService.new(portable, current_user, data).execute
+        end
+
+        def after_run(_context)
+          extractor.remove_tmpdir
+        end
+
+        private
+
+        def allowed_attributes
+          Gitlab::ImportExport::Config.new(
+            config: Gitlab::ImportExport.group_config_file
+          ).to_h.dig(:included_attributes, :namespace_settings)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb
index 97a423b6ea9..c4db53424fd 100644
--- a/lib/bulk_imports/groups/stage.rb
+++ b/lib/bulk_imports/groups/stage.rb
@@ -11,10 +11,18 @@ module BulkImports
             pipeline: BulkImports::Groups::Pipelines::GroupPipeline,
             stage: 0
           },
+          group_attributes: {
+            pipeline: BulkImports::Groups::Pipelines::GroupAttributesPipeline,
+            stage: 1
+          },
           subgroups: {
             pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline,
             stage: 1
           },
+          namespace_settings: {
+            pipeline: BulkImports::Groups::Pipelines::NamespaceSettingsPipeline,
+            stage: 1
+          },
           members: {
             pipeline: BulkImports::Common::Pipelines::MembersPipeline,
             stage: 1
@@ -69,9 +77,9 @@ module BulkImports
         if destination_namespace.present?
           root_ancestor = Namespace.find_by_full_path(destination_namespace)&.root_ancestor
 
-          ::Feature.enabled?(:bulk_import_projects, root_ancestor, default_enabled: :yaml)
+          ::Feature.enabled?(:bulk_import_projects, root_ancestor)
         else
-          ::Feature.enabled?(:bulk_import_projects, default_enabled: :yaml)
+          ::Feature.enabled?(:bulk_import_projects)
         end
       end
     end
diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
index 23e898a7bb2..df27275b664 100644
--- a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
+++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
@@ -8,7 +8,6 @@ module BulkImports
           import_entity = context.entity
 
           data
-            .then { |data| transform_name(import_entity, data) }
             .then { |data| transform_path(import_entity, data) }
             .then { |data| transform_full_path(data) }
             .then { |data| transform_parent(context, import_entity, data) }
@@ -19,11 +18,6 @@ module BulkImports
 
         private
 
-        def transform_name(import_entity, data)
-          data['name'] = import_entity.destination_name
-          data
-        end
-
         def transform_path(import_entity, data)
           data['path'] = import_entity.destination_name.parameterize
           data
diff --git a/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb
index 676a6ca8d2a..d8fb937ecd2 100644
--- a/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb
+++ b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb
@@ -8,7 +8,7 @@ module BulkImports
           {
             source_type: :group_entity,
             source_full_path: entry['full_path'],
-            destination_name: entry['name'],
+            destination_name: entry['path'],
             destination_namespace: context.entity.group.full_path,
             parent_id: context.entity.id
           }
diff --git a/lib/bulk_imports/ndjson_pipeline.rb b/lib/bulk_imports/ndjson_pipeline.rb
index d85e51984df..05d724a5e42 100644
--- a/lib/bulk_imports/ndjson_pipeline.rb
+++ b/lib/bulk_imports/ndjson_pipeline.rb
@@ -7,7 +7,7 @@ module BulkImports
     include Pipeline
 
     included do
-      ndjson_pipeline!
+      file_extraction_pipeline!
 
       def transform(context, data)
         return unless data
diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb
index 6798936576b..dc2ebdddd14 100644
--- a/lib/bulk_imports/pipeline.rb
+++ b/lib/bulk_imports/pipeline.rb
@@ -170,12 +170,12 @@ module BulkImports
         class_attributes[:abort_on_failure]
       end
 
-      def ndjson_pipeline!
-        class_attributes[:ndjson_pipeline] = true
+      def file_extraction_pipeline!
+        class_attributes[:file_extraction_pipeline] = true
       end
 
-      def ndjson_pipeline?
-        class_attributes[:ndjson_pipeline]
+      def file_extraction_pipeline?
+        class_attributes[:file_extraction_pipeline]
       end
 
       def relation_name(name)
diff --git a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb
index 2492a023cbe..1754f27137c 100644
--- a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb
+++ b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb
@@ -6,6 +6,10 @@ module BulkImports
       class ProjectAttributesPipeline
         include Pipeline
 
+        file_extraction_pipeline!
+
+        relation_name BulkImports::FileTransfer::BaseConfig::SELF_RELATION
+
         transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer
 
         def extract(_context)
@@ -55,7 +59,7 @@ module BulkImports
         def download_service
           @download_service ||= BulkImports::FileDownloadService.new(
             configuration: context.configuration,
-            relative_url:  context.entity.relation_download_url_path(BulkImports::FileTransfer::BaseConfig::SELF_RELATION),
+            relative_url:  context.entity.relation_download_url_path(self.class.relation),
             tmpdir: tmpdir,
             filename: compressed_filename
           )
@@ -70,7 +74,7 @@ module BulkImports
         end
 
         def filename
-          "#{BulkImports::FileTransfer::BaseConfig::SELF_RELATION}.json"
+          "#{self.class.relation}.json"
         end
 
         def json_decode(string)
diff --git a/lib/bulk_imports/projects/pipelines/releases_pipeline.rb b/lib/bulk_imports/projects/pipelines/releases_pipeline.rb
new file mode 100644
index 00000000000..8f9c6a5749f
--- /dev/null
+++ b/lib/bulk_imports/projects/pipelines/releases_pipeline.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Projects
+    module Pipelines
+      class ReleasesPipeline
+        include NdjsonPipeline
+
+        relation_name 'releases'
+
+        extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb
index b920c1bf355..229df9c410d 100644
--- a/lib/bulk_imports/projects/stage.rb
+++ b/lib/bulk_imports/projects/stage.rb
@@ -63,10 +63,6 @@ module BulkImports
             pipeline: BulkImports::Projects::Pipelines::ProtectedBranchesPipeline,
             stage: 4
           },
-          ci_pipelines: {
-            pipeline: BulkImports::Projects::Pipelines::CiPipelinesPipeline,
-            stage: 4
-          },
           project_feature: {
             pipeline: BulkImports::Projects::Pipelines::ProjectFeaturePipeline,
             stage: 4
@@ -79,6 +75,14 @@ module BulkImports
             pipeline: BulkImports::Projects::Pipelines::ServiceDeskSettingPipeline,
             stage: 4
           },
+          releases: {
+            pipeline: BulkImports::Projects::Pipelines::ReleasesPipeline,
+            stage: 4
+          },
+          ci_pipelines: {
+            pipeline: BulkImports::Projects::Pipelines::CiPipelinesPipeline,
+            stage: 5
+          },
           wiki: {
             pipeline: BulkImports::Common::Pipelines::WikiPipeline,
             stage: 5