From d9ab72d6080f594d0b3cae15f14b3ef2c6c638cb Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 20 Oct 2021 08:43:02 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-4-stable-ee --- app/models/bulk_imports/entity.rb | 14 ++++++++-- app/models/bulk_imports/export.rb | 2 +- app/models/bulk_imports/export_status.rb | 2 +- .../bulk_imports/file_transfer/base_config.rb | 32 ++++++++++++++++------ .../bulk_imports/file_transfer/group_config.rb | 6 ++-- .../bulk_imports/file_transfer/project_config.rb | 15 +++++++--- app/models/bulk_imports/tracker.rb | 2 ++ 7 files changed, 52 insertions(+), 21 deletions(-) (limited to 'app/models/bulk_imports') diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index ab5d248ff8c..ecac4ab95f4 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -20,6 +20,8 @@ class BulkImports::Entity < ApplicationRecord self.table_name = 'bulk_import_entities' + EXPORT_RELATIONS_URL = '/%{resource}/%{full_path}/export_relations' + belongs_to :bulk_import, optional: false belongs_to :parent, class_name: 'BulkImports::Entity', optional: true @@ -81,9 +83,9 @@ class BulkImports::Entity < ApplicationRecord def pipelines @pipelines ||= case source_type when 'group_entity' - BulkImports::Groups::Stage.pipelines + BulkImports::Groups::Stage.new(bulk_import).pipelines when 'project_entity' - BulkImports::Projects::Stage.pipelines + BulkImports::Projects::Stage.new(bulk_import).pipelines end end @@ -102,6 +104,14 @@ class BulkImports::Entity < ApplicationRecord end end + def pluralized_name + source_type.gsub('_entity', '').pluralize + end + + def export_relations_url_path + @export_relations_url_path ||= EXPORT_RELATIONS_URL % { resource: pluralized_name, full_path: encoded_source_full_path } + end + private def validate_parent_is_a_group diff --git a/app/models/bulk_imports/export.rb b/app/models/bulk_imports/export.rb index 371b58dea03..8d4d31ee92d 100644 --- a/app/models/bulk_imports/export.rb +++ b/app/models/bulk_imports/export.rb @@ -53,7 +53,7 @@ module BulkImports end def relation_definition - config.portable_tree[:include].find { |include| include[relation.to_sym] } + config.relation_definition_for(relation) end def config diff --git a/app/models/bulk_imports/export_status.rb b/app/models/bulk_imports/export_status.rb index ff165830cf1..abf064adaae 100644 --- a/app/models/bulk_imports/export_status.rb +++ b/app/models/bulk_imports/export_status.rb @@ -41,7 +41,7 @@ module BulkImports end def status_endpoint - "/groups/#{entity.encoded_source_full_path}/export_relations/status" + File.join(entity.export_relations_url_path, 'status') end end end diff --git a/app/models/bulk_imports/file_transfer/base_config.rb b/app/models/bulk_imports/file_transfer/base_config.rb index ddea7c3f64c..4d370315ad5 100644 --- a/app/models/bulk_imports/file_transfer/base_config.rb +++ b/app/models/bulk_imports/file_transfer/base_config.rb @@ -22,15 +22,25 @@ module BulkImports end def export_path - strong_memoize(:export_path) do - relative_path = File.join(base_export_path, SecureRandom.hex) - - ::Gitlab::ImportExport.export_path(relative_path: relative_path) - end + @export_path ||= Dir.mktmpdir('bulk_imports') end def portable_relations - import_export_config.dig(:tree, portable_class_sym).keys.map(&:to_s) - skipped_relations + tree_relations + file_relations - skipped_relations + end + + def tree_relation?(relation) + tree_relations.include?(relation) + end + + def file_relation?(relation) + file_relations.include?(relation) + end + + def tree_relation_definition_for(relation) + return unless tree_relation?(relation) + + portable_tree[:include].find { |include| include[relation.to_sym] } end private @@ -44,7 +54,7 @@ module BulkImports end def import_export_config - ::Gitlab::ImportExport::Config.new(config: import_export_yaml).to_h + @config ||= ::Gitlab::ImportExport::Config.new(config: import_export_yaml).to_h end def portable_class @@ -63,8 +73,12 @@ module BulkImports raise NotImplementedError end - def base_export_path - raise NotImplementedError + def tree_relations + import_export_config.dig(:tree, portable_class_sym).keys.map(&:to_s) + end + + def file_relations + [] end def skipped_relations diff --git a/app/models/bulk_imports/file_transfer/group_config.rb b/app/models/bulk_imports/file_transfer/group_config.rb index 2266cbb484f..6766c00246b 100644 --- a/app/models/bulk_imports/file_transfer/group_config.rb +++ b/app/models/bulk_imports/file_transfer/group_config.rb @@ -3,16 +3,14 @@ module BulkImports module FileTransfer class GroupConfig < BaseConfig - def base_export_path - portable.full_path - end + SKIPPED_RELATIONS = %w(members).freeze def import_export_yaml ::Gitlab::ImportExport.group_config_file end def skipped_relations - @skipped_relations ||= %w(members) + SKIPPED_RELATIONS end end end diff --git a/app/models/bulk_imports/file_transfer/project_config.rb b/app/models/bulk_imports/file_transfer/project_config.rb index 8a57f51c1c5..9a0434da08a 100644 --- a/app/models/bulk_imports/file_transfer/project_config.rb +++ b/app/models/bulk_imports/file_transfer/project_config.rb @@ -3,16 +3,23 @@ module BulkImports module FileTransfer class ProjectConfig < BaseConfig - def base_export_path - portable.disk_path - end + UPLOADS_RELATION = 'uploads' + + SKIPPED_RELATIONS = %w( + project_members + group_members + ).freeze def import_export_yaml ::Gitlab::ImportExport.config_file end + def file_relations + [UPLOADS_RELATION] + end + def skipped_relations - @skipped_relations ||= %w(project_members group_members) + SKIPPED_RELATIONS end end end diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb index c185470b1c2..9de3239ee0f 100644 --- a/app/models/bulk_imports/tracker.rb +++ b/app/models/bulk_imports/tracker.rb @@ -50,6 +50,8 @@ class BulkImports::Tracker < ApplicationRecord event :start do transition created: :started + # To avoid errors when re-starting a pipeline in case of network errors + transition started: :started end event :finish do -- cgit v1.2.1