summaryrefslogtreecommitdiff
path: root/lib/gitlab/import_export
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2019-11-19 22:11:55 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2019-11-19 22:11:55 +0000
commit5a8431feceba47fd8e1804d9aa1b1730606b71d5 (patch)
treee5df8e0ceee60f4af8093f5c4c2f934b8abced05 /lib/gitlab/import_export
parent4d477238500c347c6553d335d920bedfc5a46869 (diff)
downloadgitlab-ce-5a8431feceba47fd8e1804d9aa1b1730606b71d5.tar.gz
Add latest changes from gitlab-org/gitlab@12-5-stable-ee
Diffstat (limited to 'lib/gitlab/import_export')
-rw-r--r--lib/gitlab/import_export/config.rb5
-rw-r--r--lib/gitlab/import_export/file_importer.rb2
-rw-r--r--lib/gitlab/import_export/group_import_export.yml36
-rw-r--r--lib/gitlab/import_export/group_project_object_builder.rb11
-rw-r--r--lib/gitlab/import_export/group_tree_saver.rb55
-rw-r--r--lib/gitlab/import_export/import_export.yml3
-rw-r--r--lib/gitlab/import_export/project_tree_restorer.rb206
-rw-r--r--lib/gitlab/import_export/project_tree_saver.rb31
-rw-r--r--lib/gitlab/import_export/reader.rb27
-rw-r--r--lib/gitlab/import_export/relation_factory.rb15
-rw-r--r--lib/gitlab/import_export/relation_rename_service.rb2
-rw-r--r--lib/gitlab/import_export/relation_tree_saver.rb27
-rw-r--r--lib/gitlab/import_export/saver.rb18
-rw-r--r--lib/gitlab/import_export/shared.rb40
14 files changed, 298 insertions, 180 deletions
diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb
index 6f4919ead4e..83c4bc47349 100644
--- a/lib/gitlab/import_export/config.rb
+++ b/lib/gitlab/import_export/config.rb
@@ -3,7 +3,8 @@
module Gitlab
module ImportExport
class Config
- def initialize
+ def initialize(config: Gitlab::ImportExport.config_file)
+ @config = config
@hash = parse_yaml
@hash.deep_symbolize_keys!
@ee_hash = @hash.delete(:ee) || {}
@@ -50,7 +51,7 @@ module Gitlab
end
def parse_yaml
- YAML.load_file(Gitlab::ImportExport.config_file)
+ YAML.load_file(@config)
end
end
end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 05432f433e7..2fd12e3aa78 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -60,7 +60,7 @@ module Gitlab
def copy_archive
return if @archive_file
- @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
+ @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @project))
download_or_copy_upload(@project.import_export_upload.import_file, @archive_file)
end
diff --git a/lib/gitlab/import_export/group_import_export.yml b/lib/gitlab/import_export/group_import_export.yml
new file mode 100644
index 00000000000..c1900350c86
--- /dev/null
+++ b/lib/gitlab/import_export/group_import_export.yml
@@ -0,0 +1,36 @@
+# Model relationships to be included in the group import/export
+#
+# This list _must_ only contain relationships that are available to both FOSS and
+# Enterprise editions. EE specific relationships must be defined in the `ee` section further
+# down below.
+tree:
+ group:
+ - :milestones
+ - :badges
+ - labels:
+ - :priorities
+ - :boards
+ - members:
+ - :user
+
+included_attributes:
+
+excluded_attributes:
+ group:
+ - :runners_token
+ - :runners_token_encrypted
+
+methods:
+ labels:
+ - :type
+ badges:
+ - :type
+
+preloads:
+
+# EE specific relationships and settings to include. All of this will be merged
+# into the previous structures if EE is used.
+ee:
+ tree:
+ group:
+ - :epics
diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb
index de1629d0e28..b94839363df 100644
--- a/lib/gitlab/import_export/group_project_object_builder.rb
+++ b/lib/gitlab/import_export/group_project_object_builder.rb
@@ -49,11 +49,12 @@ module Gitlab
].compact
end
- # Returns Arel clause `"{table_name}"."project_id" = {project.id}`
+ # Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present
+ # For example: merge_request has :target_project_id, and we are searching by :iid
# or, if group is present:
# `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}`
def where_clause_base
- clause = table[:project_id].eq(project.id)
+ clause = table[:project_id].eq(project.id) if project
clause = clause.or(table[:group_id].eq(group.id)) if group
clause
@@ -103,6 +104,10 @@ module Gitlab
klass == Milestone
end
+ def merge_request?
+ klass == MergeRequest
+ end
+
# If an existing group milestone used the IID
# claim the IID back and set the group milestone to use one available
# This is necessary to fix situations like the following:
@@ -124,7 +129,7 @@ module Gitlab
# Returns Arel clause for a particular model or `nil`.
def where_clause_for_klass
- # no-op
+ return attrs_to_arel(attributes.slice('iid')) if merge_request?
end
end
end
diff --git a/lib/gitlab/import_export/group_tree_saver.rb b/lib/gitlab/import_export/group_tree_saver.rb
new file mode 100644
index 00000000000..8d2fb881cc0
--- /dev/null
+++ b/lib/gitlab/import_export/group_tree_saver.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ class GroupTreeSaver
+ attr_reader :full_path
+
+ def initialize(group:, current_user:, shared:, params: {})
+ @params = params
+ @current_user = current_user
+ @shared = shared
+ @group = group
+ @full_path = File.join(@shared.export_path, ImportExport.group_filename)
+ end
+
+ def save
+ group_tree = serialize(@group, reader.group_tree)
+ tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename)
+
+ true
+ rescue => e
+ @shared.error(e)
+ false
+ end
+
+ private
+
+ def serialize(group, relations_tree)
+ group_tree = tree_saver.serialize(group, relations_tree)
+
+ group.children.each do |child|
+ group_tree['children'] ||= []
+ group_tree['children'] << serialize(child, relations_tree)
+ end
+
+ group_tree
+ rescue => e
+ @shared.error(e)
+ end
+
+ def reader
+ @reader ||= Gitlab::ImportExport::Reader.new(
+ shared: @shared,
+ config: Gitlab::ImportExport::Config.new(
+ config: Gitlab::ImportExport.group_config_file
+ ).to_h
+ )
+ end
+
+ def tree_saver
+ @tree_saver ||= RelationTreeSaver.new
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 141e73e6a47..1aafe5804c0 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -28,6 +28,7 @@ tree:
- label:
- :priorities
- :issue_assignees
+ - :zoom_meetings
- snippets:
- :award_emoji
- notes:
@@ -147,6 +148,8 @@ excluded_attributes:
- :emails_disabled
- :max_pages_size
- :max_artifacts_size
+ - :marked_for_deletion_at
+ - :marked_for_deletion_by_user_id
namespaces:
- :runners_token
- :runners_token_encrypted
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 3fa5765fd4a..c401f96b5c1 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -15,7 +15,6 @@ module Gitlab
@user = user
@shared = shared
@project = project
- @saved = true
end
def restore
@@ -33,7 +32,8 @@ module Gitlab
ActiveRecord::Base.uncached do
ActiveRecord::Base.no_touching do
update_project_params!
- create_relations
+ create_project_relations!
+ post_import!
end
end
@@ -69,81 +69,75 @@ module Gitlab
# in the DB. The structure and relationships between models are guessed from
# the configuration yaml file too.
# Finally, it updates each attribute in the newly imported project.
- def create_relations
- project_relations.each do |relation_key, relation_definition|
- relation_key_s = relation_key.to_s
-
- if relation_definition.present?
- create_sub_relations(relation_key_s, relation_definition, @tree_hash)
- elsif @tree_hash[relation_key_s].present?
- save_relation_hash(relation_key_s, @tree_hash[relation_key_s])
- end
- end
+ def create_project_relations!
+ project_relations.each(&method(
+ :process_project_relation!))
+ end
+ def post_import!
@project.merge_requests.set_latest_merge_request_diff_ids!
-
- @saved
end
- def save_relation_hash(relation_key, relation_hash_batch)
- relation_hash = create_relation(relation_key, relation_hash_batch)
+ def process_project_relation!(relation_key, relation_definition)
+ data_hashes = @tree_hash.delete(relation_key)
+ return unless data_hashes
- remove_group_models(relation_hash) if relation_hash.is_a?(Array)
+ # we do not care if we process array or hash
+ data_hashes = [data_hashes] unless data_hashes.is_a?(Array)
- @saved = false unless @project.append_or_update_attribute(relation_key, relation_hash)
+ # consume and remove objects from memory
+ while data_hash = data_hashes.shift
+ process_project_relation_item!(relation_key, relation_definition, data_hash)
+ end
+ end
- save_id_mappings(relation_key, relation_hash_batch, relation_hash)
+ def process_project_relation_item!(relation_key, relation_definition, data_hash)
+ relation_object = build_relation(relation_key, relation_definition, data_hash)
+ return unless relation_object
+ return if group_model?(relation_object)
- @project.reset
+ relation_object.project = @project
+ relation_object.save!
+
+ save_id_mapping(relation_key, data_hash, relation_object)
end
# Older, serialized CI pipeline exports may only have a
# merge_request_id and not the full hash of the merge request. To
# import these pipelines, we need to preserve the mapping between
# the old and new the merge request ID.
- def save_id_mappings(relation_key, relation_hash_batch, relation_hash)
+ def save_id_mapping(relation_key, data_hash, relation_object)
return unless relation_key == 'merge_requests'
- relation_hash = Array(relation_hash)
-
- Array(relation_hash_batch).each_with_index do |raw_data, index|
- merge_requests_mapping[raw_data['id']] = relation_hash[index]['id']
- end
- end
-
- # Remove project models that became group models as we found them at group level.
- # This no longer required saving them at the root project level.
- # For example, in the case of an existing group label that matched the title.
- def remove_group_models(relation_hash)
- relation_hash.reject! do |value|
- GROUP_MODELS.include?(value.class) && value.group_id
- end
- end
-
- def remove_feature_dependent_sub_relations!(_relation_item)
- # no-op
+ merge_requests_mapping[data_hash['id']] = relation_object.id
end
def project_relations
- @project_relations ||= reader.attributes_finder.find_relations_tree(:project)
+ @project_relations ||=
+ reader
+ .attributes_finder
+ .find_relations_tree(:project)
+ .deep_stringify_keys
end
def update_project_params!
- Gitlab::Timeless.timeless(@project) do
- project_params = @tree_hash.reject do |key, value|
- project_relations.include?(key.to_sym)
- end
+ project_params = @tree_hash.reject do |key, value|
+ project_relations.include?(key)
+ end
- project_params = project_params.merge(present_project_override_params)
+ project_params = project_params.merge(
+ present_project_override_params)
- # Cleaning all imported and overridden params
- project_params = Gitlab::ImportExport::AttributeCleaner.clean(
- relation_hash: project_params,
- relation_class: Project,
- excluded_keys: excluded_keys_for_relation(:project))
+ # Cleaning all imported and overridden params
+ project_params = Gitlab::ImportExport::AttributeCleaner.clean(
+ relation_hash: project_params,
+ relation_class: Project,
+ excluded_keys: excluded_keys_for_relation(:project))
- @project.assign_attributes(project_params)
- @project.drop_visibility_level!
+ @project.assign_attributes(project_params)
+ @project.drop_visibility_level!
+
+ Gitlab::Timeless.timeless(@project) do
@project.save!
end
end
@@ -160,75 +154,61 @@ module Gitlab
@project_override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {}
end
- # Given a relation hash containing one or more models and its relationships,
- # loops through each model and each object from a model type and
- # and assigns its correspondent attributes hash from +tree_hash+
- # Example:
- # +relation_key+ issues, loops through the list of *issues* and for each individual
- # issue, finds any subrelations such as notes, creates them and assign them back to the hash
- #
- # Recursively calls this method if the sub-relation is a hash containing more sub-relations
- def create_sub_relations(relation_key, relation_definition, tree_hash, save: true)
- return if tree_hash[relation_key].blank?
-
- tree_array = [tree_hash[relation_key]].flatten
-
- # Avoid keeping a possible heavy object in memory once we are done with it
- while relation_item = tree_array.shift
- remove_feature_dependent_sub_relations!(relation_item)
-
- # The transaction at this level is less speedy than one single transaction
- # But we can't have it in the upper level or GC won't get rid of the AR objects
- # after we save the batch.
- Project.transaction do
- process_sub_relation(relation_key, relation_definition, relation_item)
-
- # For every subrelation that hangs from Project, save the associated records altogether
- # This effectively batches all records per subrelation item, only keeping those in memory
- # We have to keep in mind that more batch granularity << Memory, but >> Slowness
- if save
- save_relation_hash(relation_key, [relation_item])
- tree_hash[relation_key].delete(relation_item)
- end
- end
- end
-
- tree_hash.delete(relation_key) if save
+ def build_relations(relation_key, relation_definition, data_hashes)
+ data_hashes.map do |data_hash|
+ build_relation(relation_key, relation_definition, data_hash)
+ end.compact
end
- def process_sub_relation(relation_key, relation_definition, relation_item)
- relation_definition.each do |sub_relation_key, sub_relation_definition|
- # We just use author to get the user ID, do not attempt to create an instance.
- next if sub_relation_key == :author
+ def build_relation(relation_key, relation_definition, data_hash)
+ # TODO: This is hack to not create relation for the author
+ # Rather make `RelationFactory#set_note_author` to take care of that
+ return data_hash if relation_key == 'author'
- sub_relation_key_s = sub_relation_key.to_s
+ # create relation objects recursively for all sub-objects
+ relation_definition.each do |sub_relation_key, sub_relation_definition|
+ transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
+ end
- # create dependent relations if present
- if sub_relation_definition.present?
- create_sub_relations(sub_relation_key_s, sub_relation_definition, relation_item, save: false)
+ Gitlab::ImportExport::RelationFactory.create(
+ relation_sym: relation_key.to_sym,
+ relation_hash: data_hash,
+ members_mapper: members_mapper,
+ merge_requests_mapping: merge_requests_mapping,
+ user: @user,
+ project: @project,
+ excluded_keys: excluded_keys_for_relation(relation_key))
+ end
+
+ def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
+ sub_data_hash = data_hash[sub_relation_key]
+ return unless sub_data_hash
+
+ # if object is a hash we can create simple object
+ # as it means that this is 1-to-1 vs 1-to-many
+ sub_data_hash =
+ if sub_data_hash.is_a?(Array)
+ build_relations(
+ sub_relation_key,
+ sub_relation_definition,
+ sub_data_hash).presence
+ else
+ build_relation(
+ sub_relation_key,
+ sub_relation_definition,
+ sub_data_hash)
end
- # transform relation hash to actual object
- sub_relation_hash = relation_item[sub_relation_key_s]
- if sub_relation_hash.present?
- relation_item[sub_relation_key_s] = create_relation(sub_relation_key, sub_relation_hash)
- end
+ # persist object(s) or delete from relation
+ if sub_data_hash
+ data_hash[sub_relation_key] = sub_data_hash
+ else
+ data_hash.delete(sub_relation_key)
end
end
- def create_relation(relation_key, relation_hash_list)
- relation_array = [relation_hash_list].flatten.map do |relation_hash|
- Gitlab::ImportExport::RelationFactory.create(
- relation_sym: relation_key.to_sym,
- relation_hash: relation_hash,
- members_mapper: members_mapper,
- merge_requests_mapping: merge_requests_mapping,
- user: @user,
- project: @project,
- excluded_keys: excluded_keys_for_relation(relation_key))
- end.compact
-
- relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
+ def group_model?(relation_object)
+ GROUP_MODELS.include?(relation_object.class) && relation_object.group_id
end
def reader
@@ -241,5 +221,3 @@ module Gitlab
end
end
end
-
-Gitlab::ImportExport::ProjectTreeRestorer.prepend_if_ee('::EE::Gitlab::ImportExport::ProjectTreeRestorer')
diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb
index 63c71105efe..386a4cfdfc6 100644
--- a/lib/gitlab/import_export/project_tree_saver.rb
+++ b/lib/gitlab/import_export/project_tree_saver.rb
@@ -3,25 +3,20 @@
module Gitlab
module ImportExport
class ProjectTreeSaver
- include Gitlab::ImportExport::CommandLineUtil
-
attr_reader :full_path
def initialize(project:, current_user:, shared:, params: {})
- @params = params
- @project = project
+ @params = params
+ @project = project
@current_user = current_user
- @shared = shared
- @full_path = File.join(@shared.export_path, ImportExport.project_filename)
+ @shared = shared
+ @full_path = File.join(@shared.export_path, ImportExport.project_filename)
end
def save
- mkdir_p(@shared.export_path)
-
- project_tree = serialize_project_tree
+ project_tree = tree_saver.serialize(@project, reader.project_tree)
fix_project_tree(project_tree)
- project_tree_json = JSON.generate(project_tree)
- File.write(full_path, project_tree_json)
+ tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
true
rescue => e
@@ -43,16 +38,6 @@ module Gitlab
RelationRenameService.add_new_associations(project_tree)
end
- def serialize_project_tree
- if Feature.enabled?(:export_fast_serialize, default_enabled: true)
- Gitlab::ImportExport::FastHashSerializer
- .new(@project, reader.project_tree)
- .execute
- else
- @project.as_json(reader.project_tree)
- end
- end
-
def reader
@reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
end
@@ -74,6 +59,10 @@ module Gitlab
GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
end
+
+ def tree_saver
+ @tree_saver ||= RelationTreeSaver.new
+ end
end
end
end
diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb
index 9e81c6a3d07..1390770acef 100644
--- a/lib/gitlab/import_export/reader.rb
+++ b/lib/gitlab/import_export/reader.rb
@@ -5,24 +5,31 @@ module Gitlab
class Reader
attr_reader :tree, :attributes_finder
- def initialize(shared:)
- @shared = shared
-
- @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(
- config: ImportExport::Config.new.to_h)
+ def initialize(shared:, config: ImportExport::Config.new.to_h)
+ @shared = shared
+ @config = config
+ @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(config: @config)
end
# Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
# for outputting a project in JSON format, including its relations and sub relations.
def project_tree
- attributes_finder.find_root(:project)
- rescue => e
- @shared.error(e)
- false
+ tree_by_key(:project)
+ end
+
+ def group_tree
+ tree_by_key(:group)
end
def group_members_tree
- attributes_finder.find_root(:group_members)
+ tree_by_key(:group_members)
+ end
+
+ def tree_by_key(key)
+ attributes_finder.find_root(key)
+ rescue => e
+ @shared.error(e)
+ false
end
end
end
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index ae8025c52ef..ae6b3c161ce 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -38,10 +38,13 @@ module Gitlab
IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
- EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature merge_request].freeze
+ EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature merge_request ProjectCiCdSetting].freeze
TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
+ # This represents all relations that have unique key on `project_id`
+ UNIQUE_RELATIONS = %i[project_feature ProjectCiCdSetting].freeze
+
def self.create(*args)
new(*args).create
end
@@ -274,7 +277,7 @@ module Gitlab
end
def setup_pipeline
- @relation_hash.fetch('stages').each do |stage|
+ @relation_hash.fetch('stages', []).each do |stage|
stage.statuses.each do |status|
status.pipeline = imported_object
end
@@ -324,8 +327,7 @@ module Gitlab
end
def find_or_create_object!
- return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature
- return find_or_create_merge_request! if @relation_name == :merge_request
+ return relation_class.find_or_create_by(project_id: @project.id) if UNIQUE_RELATIONS.include?(@relation_name)
# Can't use IDs as validation exists calling `group` or `project` attributes
finder_hash = parsed_relation_hash.tap do |hash|
@@ -336,11 +338,6 @@ module Gitlab
GroupProjectObjectBuilder.build(relation_class, finder_hash)
end
-
- def find_or_create_merge_request!
- @project.merge_requests.find_by(iid: parsed_relation_hash['iid']) ||
- relation_class.new(parsed_relation_hash)
- end
end
end
end
diff --git a/lib/gitlab/import_export/relation_rename_service.rb b/lib/gitlab/import_export/relation_rename_service.rb
index 179bde5e21e..03aaa6aefc3 100644
--- a/lib/gitlab/import_export/relation_rename_service.rb
+++ b/lib/gitlab/import_export/relation_rename_service.rb
@@ -8,7 +8,7 @@
# The behavior of these renamed relationships should be transient and it should
# only last one release until you completely remove the renaming from the list.
#
-# When importing, this class will check the project hash and:
+# When importing, this class will check the hash and:
# - if only the old relationship name is found, it will rename it with the new one
# - if only the new relationship name is found, it will do nothing
# - if it finds both, it will use the new relationship data
diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/relation_tree_saver.rb
new file mode 100644
index 00000000000..a0452071ccf
--- /dev/null
+++ b/lib/gitlab/import_export/relation_tree_saver.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ class RelationTreeSaver
+ include Gitlab::ImportExport::CommandLineUtil
+
+ def serialize(exportable, relations_tree)
+ if Feature.enabled?(:export_fast_serialize, default_enabled: true)
+ Gitlab::ImportExport::FastHashSerializer
+ .new(exportable, relations_tree)
+ .execute
+ else
+ exportable.as_json(relations_tree)
+ end
+ end
+
+ def save(tree, dir_path, filename)
+ mkdir_p(dir_path)
+
+ tree_json = JSON.generate(tree)
+
+ File.write(File.join(dir_path, filename), tree_json)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index bea7a7cce65..ae82c380755 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -9,16 +9,16 @@ module Gitlab
new(*args).save
end
- def initialize(project:, shared:)
- @project = project
- @shared = shared
+ def initialize(exportable:, shared:)
+ @exportable = exportable
+ @shared = shared
end
def save
if compress_and_save
remove_export_path
- Rails.logger.info("Saved project export #{archive_file}") # rubocop:disable Gitlab/RailsLogger
+ Rails.logger.info("Saved #{@exportable.class} export #{archive_file}") # rubocop:disable Gitlab/RailsLogger
save_upload
else
@@ -48,11 +48,11 @@ module Gitlab
end
def archive_file
- @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
+ @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @exportable))
end
def save_upload
- upload = ImportExportUpload.find_or_initialize_by(project: @project)
+ upload = initialize_upload
File.open(archive_file) { |file| upload.export_file = file }
@@ -62,6 +62,12 @@ module Gitlab
def error_message
"Unable to save #{archive_file} into #{@shared.export_path}."
end
+
+ def initialize_upload
+ exportable_kind = @exportable.class.name.downcase
+
+ ImportExportUpload.find_or_initialize_by(Hash[exportable_kind, @exportable])
+ end
end
end
end
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
index 02d46a1f498..2539a6828c3 100644
--- a/lib/gitlab/import_export/shared.rb
+++ b/lib/gitlab/import_export/shared.rb
@@ -23,21 +23,21 @@
module Gitlab
module ImportExport
class Shared
- attr_reader :errors, :project
+ attr_reader :errors, :exportable, :logger
LOCKS_DIRECTORY = 'locks'
- def initialize(project)
- @project = project
- @errors = []
- @logger = Gitlab::Import::Logger.build
+ def initialize(exportable)
+ @exportable = exportable
+ @errors = []
+ @logger = Gitlab::Import::Logger.build
end
def active_export_count
Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) }
end
- # The path where the project metadata and repository bundle is saved
+ # The path where the exportable metadata and repository bundle (in case of project) is saved
def export_path
@export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path)
end
@@ -84,11 +84,18 @@ module Gitlab
end
def relative_archive_path
- @relative_archive_path ||= File.join(@project.disk_path, SecureRandom.hex)
+ @relative_archive_path ||= File.join(relative_base_path, SecureRandom.hex)
end
def relative_base_path
- @project.disk_path
+ case exportable_type
+ when 'Project'
+ @exportable.disk_path
+ when 'Group'
+ @exportable.full_path
+ else
+ raise Gitlab::ImportExport::Error.new("Unsupported Exportable Type #{@exportable&.class}")
+ end
end
def log_error(details)
@@ -100,17 +107,24 @@ module Gitlab
end
def log_base_data
- {
- importer: 'Import/Export',
- import_jid: @project&.import_state&.jid,
- project_id: @project&.id,
- project_path: @project&.full_path
+ log = {
+ importer: 'Import/Export',
+ exportable_id: @exportable&.id,
+ exportable_path: @exportable&.full_path
}
+
+ log[:import_jid] = @exportable&.import_state&.jid if exportable_type == 'Project'
+
+ log
end
def filtered_error_message(message)
Projects::ImportErrorFilter.filter_message(message)
end
+
+ def exportable_type
+ @exportable.class.name
+ end
end
end
end