diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-11-11 21:06:20 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-11-11 21:06:20 +0000 |
commit | 0c3f12149372a79b825d265a6c28dc547e4a1afc (patch) | |
tree | 43bdaa20afb0b061d09c56d8507efd51d28601be /lib | |
parent | ff67e3ed08355fb2d6f6e69d4ed06cd09052e573 (diff) | |
download | gitlab-ce-0c3f12149372a79b825d265a6c28dc547e4a1afc.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/import_export.rb | 14 | ||||
-rw-r--r-- | lib/gitlab/import_export/config.rb | 5 | ||||
-rw-r--r-- | lib/gitlab/import_export/file_importer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/group_import_export.yml | 36 | ||||
-rw-r--r-- | lib/gitlab/import_export/group_tree_saver.rb | 55 | ||||
-rw-r--r-- | lib/gitlab/import_export/project_tree_saver.rb | 31 | ||||
-rw-r--r-- | lib/gitlab/import_export/reader.rb | 27 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_rename_service.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_tree_saver.rb | 27 | ||||
-rw-r--r-- | lib/gitlab/import_export/saver.rb | 18 | ||||
-rw-r--r-- | lib/gitlab/import_export/shared.rb | 40 |
11 files changed, 200 insertions, 57 deletions
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index b2ac60fe825..516e7f54a6e 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -15,7 +15,7 @@ module Gitlab end def storage_path - File.join(Settings.shared['path'], 'tmp/project_exports') + File.join(Settings.shared['path'], 'tmp/gitlab_exports') end def import_upload_path(filename:) @@ -50,8 +50,8 @@ module Gitlab 'VERSION' end - def export_filename(project:) - basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.full_path.tr('/', '_')}" + def export_filename(exportable:) + basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{exportable.full_path.tr('/', '_')}" "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" end @@ -63,6 +63,14 @@ module Gitlab def reset_tokens? true end + + def group_filename + 'group.json' + end + + def group_config_file + Rails.root.join('lib/gitlab/import_export/group_import_export.yml') + end end end diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb index 6f4919ead4e..83c4bc47349 100644 --- a/lib/gitlab/import_export/config.rb +++ b/lib/gitlab/import_export/config.rb @@ -3,7 +3,8 @@ module Gitlab module ImportExport class Config - def initialize + def initialize(config: Gitlab::ImportExport.config_file) + @config = config @hash = parse_yaml @hash.deep_symbolize_keys! @ee_hash = @hash.delete(:ee) || {} @@ -50,7 +51,7 @@ module Gitlab end def parse_yaml - YAML.load_file(Gitlab::ImportExport.config_file) + YAML.load_file(@config) end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 05432f433e7..2fd12e3aa78 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -60,7 +60,7 @@ module Gitlab def copy_archive return if @archive_file - @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) + @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @project)) download_or_copy_upload(@project.import_export_upload.import_file, @archive_file) end diff --git a/lib/gitlab/import_export/group_import_export.yml b/lib/gitlab/import_export/group_import_export.yml new file mode 100644 index 00000000000..c1900350c86 --- /dev/null +++ b/lib/gitlab/import_export/group_import_export.yml @@ -0,0 +1,36 @@ +# Model relationships to be included in the group import/export +# +# This list _must_ only contain relationships that are available to both FOSS and +# Enterprise editions. EE specific relationships must be defined in the `ee` section further +# down below. +tree: + group: + - :milestones + - :badges + - labels: + - :priorities + - :boards + - members: + - :user + +included_attributes: + +excluded_attributes: + group: + - :runners_token + - :runners_token_encrypted + +methods: + labels: + - :type + badges: + - :type + +preloads: + +# EE specific relationships and settings to include. All of this will be merged +# into the previous structures if EE is used. +ee: + tree: + group: + - :epics diff --git a/lib/gitlab/import_export/group_tree_saver.rb b/lib/gitlab/import_export/group_tree_saver.rb new file mode 100644 index 00000000000..1d42bc8d3f3 --- /dev/null +++ b/lib/gitlab/import_export/group_tree_saver.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class GroupTreeSaver + attr_reader :full_path + + def initialize(group:, current_user:, shared:, params: {}) + @params = params + @current_user = current_user + @shared = shared + @group = group + @full_path = File.join(@shared.export_path, ImportExport.group_filename) + end + + def save + group_tree = serialize(@group, reader.group_tree) + tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) + + true + rescue => e + @shared.error(e) + false + end + + private + + def serialize(group, relations_tree) + group_tree = tree_saver.serialize(group, relations_tree) + + group.descendants.each do |descendant| + group_tree['descendants'] = [] unless group_tree['descendants'] + group_tree['descendants'] << serialize(descendant, relations_tree) + end + + group_tree + rescue => e + @shared.error(e) + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new( + shared: @shared, + config: Gitlab::ImportExport::Config.new( + config: Gitlab::ImportExport.group_config_file + ).to_h + ) + end + + def tree_saver + @tree_saver ||= RelationTreeSaver.new + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 63c71105efe..386a4cfdfc6 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -3,25 +3,20 @@ module Gitlab module ImportExport class ProjectTreeSaver - include Gitlab::ImportExport::CommandLineUtil - attr_reader :full_path def initialize(project:, current_user:, shared:, params: {}) - @params = params - @project = project + @params = params + @project = project @current_user = current_user - @shared = shared - @full_path = File.join(@shared.export_path, ImportExport.project_filename) + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) end def save - mkdir_p(@shared.export_path) - - project_tree = serialize_project_tree + project_tree = tree_saver.serialize(@project, reader.project_tree) fix_project_tree(project_tree) - project_tree_json = JSON.generate(project_tree) - File.write(full_path, project_tree_json) + tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) true rescue => e @@ -43,16 +38,6 @@ module Gitlab RelationRenameService.add_new_associations(project_tree) end - def serialize_project_tree - if Feature.enabled?(:export_fast_serialize, default_enabled: true) - Gitlab::ImportExport::FastHashSerializer - .new(@project, reader.project_tree) - .execute - else - @project.as_json(reader.project_tree) - end - end - def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end @@ -74,6 +59,10 @@ module Gitlab GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) end + + def tree_saver + @tree_saver ||= RelationTreeSaver.new + end end end end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 9e81c6a3d07..1390770acef 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -5,24 +5,31 @@ module Gitlab class Reader attr_reader :tree, :attributes_finder - def initialize(shared:) - @shared = shared - - @attributes_finder = Gitlab::ImportExport::AttributesFinder.new( - config: ImportExport::Config.new.to_h) + def initialize(shared:, config: ImportExport::Config.new.to_h) + @shared = shared + @config = config + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(config: @config) end # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html # for outputting a project in JSON format, including its relations and sub relations. def project_tree - attributes_finder.find_root(:project) - rescue => e - @shared.error(e) - false + tree_by_key(:project) + end + + def group_tree + tree_by_key(:group) end def group_members_tree - attributes_finder.find_root(:group_members) + tree_by_key(:group_members) + end + + def tree_by_key(key) + attributes_finder.find_root(key) + rescue => e + @shared.error(e) + false end end end diff --git a/lib/gitlab/import_export/relation_rename_service.rb b/lib/gitlab/import_export/relation_rename_service.rb index 179bde5e21e..03aaa6aefc3 100644 --- a/lib/gitlab/import_export/relation_rename_service.rb +++ b/lib/gitlab/import_export/relation_rename_service.rb @@ -8,7 +8,7 @@ # The behavior of these renamed relationships should be transient and it should # only last one release until you completely remove the renaming from the list. # -# When importing, this class will check the project hash and: +# When importing, this class will check the hash and: # - if only the old relationship name is found, it will rename it with the new one # - if only the new relationship name is found, it will do nothing # - if it finds both, it will use the new relationship data diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/relation_tree_saver.rb new file mode 100644 index 00000000000..a0452071ccf --- /dev/null +++ b/lib/gitlab/import_export/relation_tree_saver.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class RelationTreeSaver + include Gitlab::ImportExport::CommandLineUtil + + def serialize(exportable, relations_tree) + if Feature.enabled?(:export_fast_serialize, default_enabled: true) + Gitlab::ImportExport::FastHashSerializer + .new(exportable, relations_tree) + .execute + else + exportable.as_json(relations_tree) + end + end + + def save(tree, dir_path, filename) + mkdir_p(dir_path) + + tree_json = JSON.generate(tree) + + File.write(File.join(dir_path, filename), tree_json) + end + end + end +end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index bea7a7cce65..ae82c380755 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -9,16 +9,16 @@ module Gitlab new(*args).save end - def initialize(project:, shared:) - @project = project - @shared = shared + def initialize(exportable:, shared:) + @exportable = exportable + @shared = shared end def save if compress_and_save remove_export_path - Rails.logger.info("Saved project export #{archive_file}") # rubocop:disable Gitlab/RailsLogger + Rails.logger.info("Saved #{@exportable.class} export #{archive_file}") # rubocop:disable Gitlab/RailsLogger save_upload else @@ -48,11 +48,11 @@ module Gitlab end def archive_file - @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) + @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @exportable)) end def save_upload - upload = ImportExportUpload.find_or_initialize_by(project: @project) + upload = initialize_upload File.open(archive_file) { |file| upload.export_file = file } @@ -62,6 +62,12 @@ module Gitlab def error_message "Unable to save #{archive_file} into #{@shared.export_path}." end + + def initialize_upload + exportable_kind = @exportable.class.name.downcase + + ImportExportUpload.find_or_initialize_by(Hash[exportable_kind, @exportable]) + end end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 02d46a1f498..2539a6828c3 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -23,21 +23,21 @@ module Gitlab module ImportExport class Shared - attr_reader :errors, :project + attr_reader :errors, :exportable, :logger LOCKS_DIRECTORY = 'locks' - def initialize(project) - @project = project - @errors = [] - @logger = Gitlab::Import::Logger.build + def initialize(exportable) + @exportable = exportable + @errors = [] + @logger = Gitlab::Import::Logger.build end def active_export_count Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) } end - # The path where the project metadata and repository bundle is saved + # The path where the exportable metadata and repository bundle (in case of project) is saved def export_path @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path) end @@ -84,11 +84,18 @@ module Gitlab end def relative_archive_path - @relative_archive_path ||= File.join(@project.disk_path, SecureRandom.hex) + @relative_archive_path ||= File.join(relative_base_path, SecureRandom.hex) end def relative_base_path - @project.disk_path + case exportable_type + when 'Project' + @exportable.disk_path + when 'Group' + @exportable.full_path + else + raise Gitlab::ImportExport::Error.new("Unsupported Exportable Type #{@exportable&.class}") + end end def log_error(details) @@ -100,17 +107,24 @@ module Gitlab end def log_base_data - { - importer: 'Import/Export', - import_jid: @project&.import_state&.jid, - project_id: @project&.id, - project_path: @project&.full_path + log = { + importer: 'Import/Export', + exportable_id: @exportable&.id, + exportable_path: @exportable&.full_path } + + log[:import_jid] = @exportable&.import_state&.jid if exportable_type == 'Project' + + log end def filtered_error_message(message) Projects::ImportErrorFilter.filter_message(message) end + + def exportable_type + @exportable.class.name + end end end end |