diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-11-19 22:11:55 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-11-19 22:11:55 +0000 |
commit | 5a8431feceba47fd8e1804d9aa1b1730606b71d5 (patch) | |
tree | e5df8e0ceee60f4af8093f5c4c2f934b8abced05 /lib/gitlab/import_export | |
parent | 4d477238500c347c6553d335d920bedfc5a46869 (diff) | |
download | gitlab-ce-5a8431feceba47fd8e1804d9aa1b1730606b71d5.tar.gz |
Add latest changes from gitlab-org/gitlab@12-5-stable-ee
Diffstat (limited to 'lib/gitlab/import_export')
-rw-r--r-- | lib/gitlab/import_export/config.rb | 5 | ||||
-rw-r--r-- | lib/gitlab/import_export/file_importer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/group_import_export.yml | 36 | ||||
-rw-r--r-- | lib/gitlab/import_export/group_project_object_builder.rb | 11 | ||||
-rw-r--r-- | lib/gitlab/import_export/group_tree_saver.rb | 55 | ||||
-rw-r--r-- | lib/gitlab/import_export/import_export.yml | 3 | ||||
-rw-r--r-- | lib/gitlab/import_export/project_tree_restorer.rb | 206 | ||||
-rw-r--r-- | lib/gitlab/import_export/project_tree_saver.rb | 31 | ||||
-rw-r--r-- | lib/gitlab/import_export/reader.rb | 27 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_factory.rb | 15 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_rename_service.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_tree_saver.rb | 27 | ||||
-rw-r--r-- | lib/gitlab/import_export/saver.rb | 18 | ||||
-rw-r--r-- | lib/gitlab/import_export/shared.rb | 40 |
14 files changed, 298 insertions, 180 deletions
diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb index 6f4919ead4e..83c4bc47349 100644 --- a/lib/gitlab/import_export/config.rb +++ b/lib/gitlab/import_export/config.rb @@ -3,7 +3,8 @@ module Gitlab module ImportExport class Config - def initialize + def initialize(config: Gitlab::ImportExport.config_file) + @config = config @hash = parse_yaml @hash.deep_symbolize_keys! @ee_hash = @hash.delete(:ee) || {} @@ -50,7 +51,7 @@ module Gitlab end def parse_yaml - YAML.load_file(Gitlab::ImportExport.config_file) + YAML.load_file(@config) end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 05432f433e7..2fd12e3aa78 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -60,7 +60,7 @@ module Gitlab def copy_archive return if @archive_file - @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) + @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @project)) download_or_copy_upload(@project.import_export_upload.import_file, @archive_file) end diff --git a/lib/gitlab/import_export/group_import_export.yml b/lib/gitlab/import_export/group_import_export.yml new file mode 100644 index 00000000000..c1900350c86 --- /dev/null +++ b/lib/gitlab/import_export/group_import_export.yml @@ -0,0 +1,36 @@ +# Model relationships to be included in the group import/export +# +# This list _must_ only contain relationships that are available to both FOSS and +# Enterprise editions. EE specific relationships must be defined in the `ee` section further +# down below. +tree: + group: + - :milestones + - :badges + - labels: + - :priorities + - :boards + - members: + - :user + +included_attributes: + +excluded_attributes: + group: + - :runners_token + - :runners_token_encrypted + +methods: + labels: + - :type + badges: + - :type + +preloads: + +# EE specific relationships and settings to include. All of this will be merged +# into the previous structures if EE is used. +ee: + tree: + group: + - :epics diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb index de1629d0e28..b94839363df 100644 --- a/lib/gitlab/import_export/group_project_object_builder.rb +++ b/lib/gitlab/import_export/group_project_object_builder.rb @@ -49,11 +49,12 @@ module Gitlab ].compact end - # Returns Arel clause `"{table_name}"."project_id" = {project.id}` + # Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present + # For example: merge_request has :target_project_id, and we are searching by :iid # or, if group is present: # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}` def where_clause_base - clause = table[:project_id].eq(project.id) + clause = table[:project_id].eq(project.id) if project clause = clause.or(table[:group_id].eq(group.id)) if group clause @@ -103,6 +104,10 @@ module Gitlab klass == Milestone end + def merge_request? + klass == MergeRequest + end + # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: @@ -124,7 +129,7 @@ module Gitlab # Returns Arel clause for a particular model or `nil`. def where_clause_for_klass - # no-op + return attrs_to_arel(attributes.slice('iid')) if merge_request? end end end diff --git a/lib/gitlab/import_export/group_tree_saver.rb b/lib/gitlab/import_export/group_tree_saver.rb new file mode 100644 index 00000000000..8d2fb881cc0 --- /dev/null +++ b/lib/gitlab/import_export/group_tree_saver.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class GroupTreeSaver + attr_reader :full_path + + def initialize(group:, current_user:, shared:, params: {}) + @params = params + @current_user = current_user + @shared = shared + @group = group + @full_path = File.join(@shared.export_path, ImportExport.group_filename) + end + + def save + group_tree = serialize(@group, reader.group_tree) + tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) + + true + rescue => e + @shared.error(e) + false + end + + private + + def serialize(group, relations_tree) + group_tree = tree_saver.serialize(group, relations_tree) + + group.children.each do |child| + group_tree['children'] ||= [] + group_tree['children'] << serialize(child, relations_tree) + end + + group_tree + rescue => e + @shared.error(e) + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new( + shared: @shared, + config: Gitlab::ImportExport::Config.new( + config: Gitlab::ImportExport.group_config_file + ).to_h + ) + end + + def tree_saver + @tree_saver ||= RelationTreeSaver.new + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 141e73e6a47..1aafe5804c0 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -28,6 +28,7 @@ tree: - label: - :priorities - :issue_assignees + - :zoom_meetings - snippets: - :award_emoji - notes: @@ -147,6 +148,8 @@ excluded_attributes: - :emails_disabled - :max_pages_size - :max_artifacts_size + - :marked_for_deletion_at + - :marked_for_deletion_by_user_id namespaces: - :runners_token - :runners_token_encrypted diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 3fa5765fd4a..c401f96b5c1 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -15,7 +15,6 @@ module Gitlab @user = user @shared = shared @project = project - @saved = true end def restore @@ -33,7 +32,8 @@ module Gitlab ActiveRecord::Base.uncached do ActiveRecord::Base.no_touching do update_project_params! - create_relations + create_project_relations! + post_import! end end @@ -69,81 +69,75 @@ module Gitlab # in the DB. The structure and relationships between models are guessed from # the configuration yaml file too. # Finally, it updates each attribute in the newly imported project. - def create_relations - project_relations.each do |relation_key, relation_definition| - relation_key_s = relation_key.to_s - - if relation_definition.present? - create_sub_relations(relation_key_s, relation_definition, @tree_hash) - elsif @tree_hash[relation_key_s].present? - save_relation_hash(relation_key_s, @tree_hash[relation_key_s]) - end - end + def create_project_relations! + project_relations.each(&method( + :process_project_relation!)) + end + def post_import! @project.merge_requests.set_latest_merge_request_diff_ids! - - @saved end - def save_relation_hash(relation_key, relation_hash_batch) - relation_hash = create_relation(relation_key, relation_hash_batch) + def process_project_relation!(relation_key, relation_definition) + data_hashes = @tree_hash.delete(relation_key) + return unless data_hashes - remove_group_models(relation_hash) if relation_hash.is_a?(Array) + # we do not care if we process array or hash + data_hashes = [data_hashes] unless data_hashes.is_a?(Array) - @saved = false unless @project.append_or_update_attribute(relation_key, relation_hash) + # consume and remove objects from memory + while data_hash = data_hashes.shift + process_project_relation_item!(relation_key, relation_definition, data_hash) + end + end - save_id_mappings(relation_key, relation_hash_batch, relation_hash) + def process_project_relation_item!(relation_key, relation_definition, data_hash) + relation_object = build_relation(relation_key, relation_definition, data_hash) + return unless relation_object + return if group_model?(relation_object) - @project.reset + relation_object.project = @project + relation_object.save! + + save_id_mapping(relation_key, data_hash, relation_object) end # Older, serialized CI pipeline exports may only have a # merge_request_id and not the full hash of the merge request. To # import these pipelines, we need to preserve the mapping between # the old and new the merge request ID. - def save_id_mappings(relation_key, relation_hash_batch, relation_hash) + def save_id_mapping(relation_key, data_hash, relation_object) return unless relation_key == 'merge_requests' - relation_hash = Array(relation_hash) - - Array(relation_hash_batch).each_with_index do |raw_data, index| - merge_requests_mapping[raw_data['id']] = relation_hash[index]['id'] - end - end - - # Remove project models that became group models as we found them at group level. - # This no longer required saving them at the root project level. - # For example, in the case of an existing group label that matched the title. - def remove_group_models(relation_hash) - relation_hash.reject! do |value| - GROUP_MODELS.include?(value.class) && value.group_id - end - end - - def remove_feature_dependent_sub_relations!(_relation_item) - # no-op + merge_requests_mapping[data_hash['id']] = relation_object.id end def project_relations - @project_relations ||= reader.attributes_finder.find_relations_tree(:project) + @project_relations ||= + reader + .attributes_finder + .find_relations_tree(:project) + .deep_stringify_keys end def update_project_params! - Gitlab::Timeless.timeless(@project) do - project_params = @tree_hash.reject do |key, value| - project_relations.include?(key.to_sym) - end + project_params = @tree_hash.reject do |key, value| + project_relations.include?(key) + end - project_params = project_params.merge(present_project_override_params) + project_params = project_params.merge( + present_project_override_params) - # Cleaning all imported and overridden params - project_params = Gitlab::ImportExport::AttributeCleaner.clean( - relation_hash: project_params, - relation_class: Project, - excluded_keys: excluded_keys_for_relation(:project)) + # Cleaning all imported and overridden params + project_params = Gitlab::ImportExport::AttributeCleaner.clean( + relation_hash: project_params, + relation_class: Project, + excluded_keys: excluded_keys_for_relation(:project)) - @project.assign_attributes(project_params) - @project.drop_visibility_level! + @project.assign_attributes(project_params) + @project.drop_visibility_level! + + Gitlab::Timeless.timeless(@project) do @project.save! end end @@ -160,75 +154,61 @@ module Gitlab @project_override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {} end - # Given a relation hash containing one or more models and its relationships, - # loops through each model and each object from a model type and - # and assigns its correspondent attributes hash from +tree_hash+ - # Example: - # +relation_key+ issues, loops through the list of *issues* and for each individual - # issue, finds any subrelations such as notes, creates them and assign them back to the hash - # - # Recursively calls this method if the sub-relation is a hash containing more sub-relations - def create_sub_relations(relation_key, relation_definition, tree_hash, save: true) - return if tree_hash[relation_key].blank? - - tree_array = [tree_hash[relation_key]].flatten - - # Avoid keeping a possible heavy object in memory once we are done with it - while relation_item = tree_array.shift - remove_feature_dependent_sub_relations!(relation_item) - - # The transaction at this level is less speedy than one single transaction - # But we can't have it in the upper level or GC won't get rid of the AR objects - # after we save the batch. - Project.transaction do - process_sub_relation(relation_key, relation_definition, relation_item) - - # For every subrelation that hangs from Project, save the associated records altogether - # This effectively batches all records per subrelation item, only keeping those in memory - # We have to keep in mind that more batch granularity << Memory, but >> Slowness - if save - save_relation_hash(relation_key, [relation_item]) - tree_hash[relation_key].delete(relation_item) - end - end - end - - tree_hash.delete(relation_key) if save + def build_relations(relation_key, relation_definition, data_hashes) + data_hashes.map do |data_hash| + build_relation(relation_key, relation_definition, data_hash) + end.compact end - def process_sub_relation(relation_key, relation_definition, relation_item) - relation_definition.each do |sub_relation_key, sub_relation_definition| - # We just use author to get the user ID, do not attempt to create an instance. - next if sub_relation_key == :author + def build_relation(relation_key, relation_definition, data_hash) + # TODO: This is hack to not create relation for the author + # Rather make `RelationFactory#set_note_author` to take care of that + return data_hash if relation_key == 'author' - sub_relation_key_s = sub_relation_key.to_s + # create relation objects recursively for all sub-objects + relation_definition.each do |sub_relation_key, sub_relation_definition| + transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition) + end - # create dependent relations if present - if sub_relation_definition.present? - create_sub_relations(sub_relation_key_s, sub_relation_definition, relation_item, save: false) + Gitlab::ImportExport::RelationFactory.create( + relation_sym: relation_key.to_sym, + relation_hash: data_hash, + members_mapper: members_mapper, + merge_requests_mapping: merge_requests_mapping, + user: @user, + project: @project, + excluded_keys: excluded_keys_for_relation(relation_key)) + end + + def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition) + sub_data_hash = data_hash[sub_relation_key] + return unless sub_data_hash + + # if object is a hash we can create simple object + # as it means that this is 1-to-1 vs 1-to-many + sub_data_hash = + if sub_data_hash.is_a?(Array) + build_relations( + sub_relation_key, + sub_relation_definition, + sub_data_hash).presence + else + build_relation( + sub_relation_key, + sub_relation_definition, + sub_data_hash) end - # transform relation hash to actual object - sub_relation_hash = relation_item[sub_relation_key_s] - if sub_relation_hash.present? - relation_item[sub_relation_key_s] = create_relation(sub_relation_key, sub_relation_hash) - end + # persist object(s) or delete from relation + if sub_data_hash + data_hash[sub_relation_key] = sub_data_hash + else + data_hash.delete(sub_relation_key) end end - def create_relation(relation_key, relation_hash_list) - relation_array = [relation_hash_list].flatten.map do |relation_hash| - Gitlab::ImportExport::RelationFactory.create( - relation_sym: relation_key.to_sym, - relation_hash: relation_hash, - members_mapper: members_mapper, - merge_requests_mapping: merge_requests_mapping, - user: @user, - project: @project, - excluded_keys: excluded_keys_for_relation(relation_key)) - end.compact - - relation_hash_list.is_a?(Array) ? relation_array : relation_array.first + def group_model?(relation_object) + GROUP_MODELS.include?(relation_object.class) && relation_object.group_id end def reader @@ -241,5 +221,3 @@ module Gitlab end end end - -Gitlab::ImportExport::ProjectTreeRestorer.prepend_if_ee('::EE::Gitlab::ImportExport::ProjectTreeRestorer') diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 63c71105efe..386a4cfdfc6 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -3,25 +3,20 @@ module Gitlab module ImportExport class ProjectTreeSaver - include Gitlab::ImportExport::CommandLineUtil - attr_reader :full_path def initialize(project:, current_user:, shared:, params: {}) - @params = params - @project = project + @params = params + @project = project @current_user = current_user - @shared = shared - @full_path = File.join(@shared.export_path, ImportExport.project_filename) + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) end def save - mkdir_p(@shared.export_path) - - project_tree = serialize_project_tree + project_tree = tree_saver.serialize(@project, reader.project_tree) fix_project_tree(project_tree) - project_tree_json = JSON.generate(project_tree) - File.write(full_path, project_tree_json) + tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) true rescue => e @@ -43,16 +38,6 @@ module Gitlab RelationRenameService.add_new_associations(project_tree) end - def serialize_project_tree - if Feature.enabled?(:export_fast_serialize, default_enabled: true) - Gitlab::ImportExport::FastHashSerializer - .new(@project, reader.project_tree) - .execute - else - @project.as_json(reader.project_tree) - end - end - def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end @@ -74,6 +59,10 @@ module Gitlab GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) end + + def tree_saver + @tree_saver ||= RelationTreeSaver.new + end end end end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 9e81c6a3d07..1390770acef 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -5,24 +5,31 @@ module Gitlab class Reader attr_reader :tree, :attributes_finder - def initialize(shared:) - @shared = shared - - @attributes_finder = Gitlab::ImportExport::AttributesFinder.new( - config: ImportExport::Config.new.to_h) + def initialize(shared:, config: ImportExport::Config.new.to_h) + @shared = shared + @config = config + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(config: @config) end # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html # for outputting a project in JSON format, including its relations and sub relations. def project_tree - attributes_finder.find_root(:project) - rescue => e - @shared.error(e) - false + tree_by_key(:project) + end + + def group_tree + tree_by_key(:group) end def group_members_tree - attributes_finder.find_root(:group_members) + tree_by_key(:group_members) + end + + def tree_by_key(key) + attributes_finder.find_root(key) + rescue => e + @shared.error(e) + false end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index ae8025c52ef..ae6b3c161ce 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -38,10 +38,13 @@ module Gitlab IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature merge_request].freeze + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature merge_request ProjectCiCdSetting].freeze TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze + # This represents all relations that have unique key on `project_id` + UNIQUE_RELATIONS = %i[project_feature ProjectCiCdSetting].freeze + def self.create(*args) new(*args).create end @@ -274,7 +277,7 @@ module Gitlab end def setup_pipeline - @relation_hash.fetch('stages').each do |stage| + @relation_hash.fetch('stages', []).each do |stage| stage.statuses.each do |status| status.pipeline = imported_object end @@ -324,8 +327,7 @@ module Gitlab end def find_or_create_object! - return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature - return find_or_create_merge_request! if @relation_name == :merge_request + return relation_class.find_or_create_by(project_id: @project.id) if UNIQUE_RELATIONS.include?(@relation_name) # Can't use IDs as validation exists calling `group` or `project` attributes finder_hash = parsed_relation_hash.tap do |hash| @@ -336,11 +338,6 @@ module Gitlab GroupProjectObjectBuilder.build(relation_class, finder_hash) end - - def find_or_create_merge_request! - @project.merge_requests.find_by(iid: parsed_relation_hash['iid']) || - relation_class.new(parsed_relation_hash) - end end end end diff --git a/lib/gitlab/import_export/relation_rename_service.rb b/lib/gitlab/import_export/relation_rename_service.rb index 179bde5e21e..03aaa6aefc3 100644 --- a/lib/gitlab/import_export/relation_rename_service.rb +++ b/lib/gitlab/import_export/relation_rename_service.rb @@ -8,7 +8,7 @@ # The behavior of these renamed relationships should be transient and it should # only last one release until you completely remove the renaming from the list. # -# When importing, this class will check the project hash and: +# When importing, this class will check the hash and: # - if only the old relationship name is found, it will rename it with the new one # - if only the new relationship name is found, it will do nothing # - if it finds both, it will use the new relationship data diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/relation_tree_saver.rb new file mode 100644 index 00000000000..a0452071ccf --- /dev/null +++ b/lib/gitlab/import_export/relation_tree_saver.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class RelationTreeSaver + include Gitlab::ImportExport::CommandLineUtil + + def serialize(exportable, relations_tree) + if Feature.enabled?(:export_fast_serialize, default_enabled: true) + Gitlab::ImportExport::FastHashSerializer + .new(exportable, relations_tree) + .execute + else + exportable.as_json(relations_tree) + end + end + + def save(tree, dir_path, filename) + mkdir_p(dir_path) + + tree_json = JSON.generate(tree) + + File.write(File.join(dir_path, filename), tree_json) + end + end + end +end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index bea7a7cce65..ae82c380755 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -9,16 +9,16 @@ module Gitlab new(*args).save end - def initialize(project:, shared:) - @project = project - @shared = shared + def initialize(exportable:, shared:) + @exportable = exportable + @shared = shared end def save if compress_and_save remove_export_path - Rails.logger.info("Saved project export #{archive_file}") # rubocop:disable Gitlab/RailsLogger + Rails.logger.info("Saved #{@exportable.class} export #{archive_file}") # rubocop:disable Gitlab/RailsLogger save_upload else @@ -48,11 +48,11 @@ module Gitlab end def archive_file - @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) + @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @exportable)) end def save_upload - upload = ImportExportUpload.find_or_initialize_by(project: @project) + upload = initialize_upload File.open(archive_file) { |file| upload.export_file = file } @@ -62,6 +62,12 @@ module Gitlab def error_message "Unable to save #{archive_file} into #{@shared.export_path}." end + + def initialize_upload + exportable_kind = @exportable.class.name.downcase + + ImportExportUpload.find_or_initialize_by(Hash[exportable_kind, @exportable]) + end end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 02d46a1f498..2539a6828c3 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -23,21 +23,21 @@ module Gitlab module ImportExport class Shared - attr_reader :errors, :project + attr_reader :errors, :exportable, :logger LOCKS_DIRECTORY = 'locks' - def initialize(project) - @project = project - @errors = [] - @logger = Gitlab::Import::Logger.build + def initialize(exportable) + @exportable = exportable + @errors = [] + @logger = Gitlab::Import::Logger.build end def active_export_count Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) } end - # The path where the project metadata and repository bundle is saved + # The path where the exportable metadata and repository bundle (in case of project) is saved def export_path @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path) end @@ -84,11 +84,18 @@ module Gitlab end def relative_archive_path - @relative_archive_path ||= File.join(@project.disk_path, SecureRandom.hex) + @relative_archive_path ||= File.join(relative_base_path, SecureRandom.hex) end def relative_base_path - @project.disk_path + case exportable_type + when 'Project' + @exportable.disk_path + when 'Group' + @exportable.full_path + else + raise Gitlab::ImportExport::Error.new("Unsupported Exportable Type #{@exportable&.class}") + end end def log_error(details) @@ -100,17 +107,24 @@ module Gitlab end def log_base_data - { - importer: 'Import/Export', - import_jid: @project&.import_state&.jid, - project_id: @project&.id, - project_path: @project&.full_path + log = { + importer: 'Import/Export', + exportable_id: @exportable&.id, + exportable_path: @exportable&.full_path } + + log[:import_jid] = @exportable&.import_state&.jid if exportable_type == 'Project' + + log end def filtered_error_message(message) Projects::ImportErrorFilter.filter_message(message) end + + def exportable_type + @exportable.class.name + end end end end |