diff options
Diffstat (limited to 'lib/gitlab/import_export')
21 files changed, 1001 insertions, 0 deletions
diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb new file mode 100644 index 00000000000..d230de781d5 --- /dev/null +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -0,0 +1,47 @@ +module Gitlab + module ImportExport + class AttributesFinder + + def initialize(included_attributes:, excluded_attributes:, methods:) + @included_attributes = included_attributes || {} + @excluded_attributes = excluded_attributes || {} + @methods = methods || {} + end + + def find(model_object) + parsed_hash = find_attributes_only(model_object) + parsed_hash.empty? ? model_object : { model_object => parsed_hash } + end + + def parse(model_object) + parsed_hash = find_attributes_only(model_object) + yield parsed_hash unless parsed_hash.empty? + end + + def find_included(value) + key = key_from_hash(value) + @included_attributes[key].nil? ? {} : { only: @included_attributes[key] } + end + + def find_excluded(value) + key = key_from_hash(value) + @excluded_attributes[key].nil? ? {} : { except: @excluded_attributes[key] } + end + + def find_method(value) + key = key_from_hash(value) + @methods[key].nil? ? {} : { methods: @methods[key] } + end + + private + + def find_attributes_only(value) + find_included(value).merge(find_excluded(value)).merge(find_method(value)) + end + + def key_from_hash(value) + value.is_a?(Hash) ? value.keys.first : value + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb new file mode 100644 index 00000000000..78664f076eb --- /dev/null +++ b/lib/gitlab/import_export/command_line_util.rb @@ -0,0 +1,40 @@ +module Gitlab + module ImportExport + module CommandLineUtil + def tar_czf(archive:, dir:) + tar_with_options(archive: archive, dir: dir, options: 'czf') + end + + def untar_zxf(archive:, dir:) + untar_with_options(archive: archive, dir: dir, options: 'zxf') + end + + def git_bundle(repo_path:, bundle_path:) + execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) + end + + def git_unbundle(repo_path:, bundle_path:) + execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path})) + end + + private + + def tar_with_options(archive:, dir:, options:) + execute(%W(tar -#{options} #{archive} -C #{dir} .)) + end + + def untar_with_options(archive:, dir:, options:) + execute(%W(tar -#{options} #{archive} -C #{dir})) + end + + def execute(cmd) + _output, status = Gitlab::Popen.popen(cmd) + status.zero? + end + + def git_bin_path + Gitlab.config.git.bin_path + end + end + end +end diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb new file mode 100644 index 00000000000..e341c4d9cf8 --- /dev/null +++ b/lib/gitlab/import_export/error.rb @@ -0,0 +1,5 @@ +module Gitlab + module ImportExport + class Error < StandardError; end + end +end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb new file mode 100644 index 00000000000..0e70d9282d5 --- /dev/null +++ b/lib/gitlab/import_export/file_importer.rb @@ -0,0 +1,30 @@ +module Gitlab + module ImportExport + class FileImporter + include Gitlab::ImportExport::CommandLineUtil + + def self.import(*args) + new(*args).import + end + + def initialize(archive_file:, shared:) + @archive_file = archive_file + @shared = shared + end + + def import + FileUtils.mkdir_p(@shared.export_path) + decompress_archive + rescue => e + @shared.error(e) + false + end + + private + + def decompress_archive + untar_zxf(archive: @archive_file, dir: @shared.export_path) + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml new file mode 100644 index 00000000000..164ab6238c4 --- /dev/null +++ b/lib/gitlab/import_export/import_export.yml @@ -0,0 +1,54 @@ +# Model relationships to be included in the project import/export +project_tree: + - issues: + - notes: + :author + - :labels + - :milestones + - snippets: + - notes: + :author + - :releases + - :events + - project_members: + - :user + - merge_requests: + - notes: + :author + - :merge_request_diff + - pipelines: + - notes: + :author + - :statuses + - :variables + - :triggers + - :deploy_keys + - :services + - :hooks + - :protected_branches + +# Only include the following attributes for the models specified. +included_attributes: + project: + - :description + - :issues_enabled + - :merge_requests_enabled + - :wiki_enabled + - :snippets_enabled + - :visibility_level + - :archived + user: + - :id + - :email + - :username + author: + - :name + +# Do not include the following attributes for the models specified. +excluded_attributes: + snippets: + - :expired_at + +methods: + statuses: + - :type
\ No newline at end of file diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb new file mode 100644 index 00000000000..d209e04f7be --- /dev/null +++ b/lib/gitlab/import_export/importer.rb @@ -0,0 +1,64 @@ +module Gitlab + module ImportExport + class Importer + + def initialize(project) + @archive_file = project.import_source + @current_user = project.creator + @project = project + @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace) + end + + def execute + Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + shared: @shared) + if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) + project_tree.restored_project + else + raise Projects::ImportService::Error.new(@shared.errors.join(', ')) + end + end + + private + + def check_version! + Gitlab::ImportExport::VersionChecker.check!(shared: @shared) + end + + def project_tree + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user, + shared: @shared, + project: @project) + end + + def repo_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, + shared: @shared, + project: project_tree.restored_project) + end + + def wiki_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, + shared: @shared, + project: ProjectWiki.new(project_tree.restored_project), + wiki: true) + end + + def uploads_restorer + Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + + def path_with_namespace + File.join(@project.namespace.path, @project.path) + end + + def repo_path + File.join(@shared.export_path, 'project.bundle') + end + + def wiki_repo_path + File.join(@shared.export_path, 'project.wiki.bundle') + end + end + end +end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb new file mode 100644 index 00000000000..c569a35a48b --- /dev/null +++ b/lib/gitlab/import_export/members_mapper.rb @@ -0,0 +1,68 @@ +module Gitlab + module ImportExport + class MembersMapper + + attr_reader :missing_author_ids + + def initialize(exported_members:, user:, project:) + @exported_members = exported_members + @user = user + @project = project + @missing_author_ids = [] + + # This needs to run first, as second call would be from #map + # which means project members already exist. + ensure_default_member! + end + + def map + @map ||= + begin + @exported_members.inject(missing_keys_tracking_hash) do |hash, member| + existing_user = User.where(find_project_user_query(member)).first + old_user_id = member['user']['id'] + if existing_user && add_user_as_team_member(existing_user, member) + hash[old_user_id] = existing_user.id + end + hash + end + end + end + + def default_user_id + @user.id + end + + private + + def missing_keys_tracking_hash + Hash.new do |_, key| + @missing_author_ids << key + default_user_id + end + end + + def ensure_default_member! + ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) + end + + def add_user_as_team_member(existing_user, member) + member['user'] = existing_user + + ProjectMember.create(member_hash(member)).persisted? + end + + def member_hash(member) + member.except('id').merge(source_id: @project.id, importing: true) + end + + def find_project_user_query(member) + user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email'])) + end + + def user_arel + @user_arel ||= User.arel_table + end + end + end +end diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb new file mode 100644 index 00000000000..89388d1984b --- /dev/null +++ b/lib/gitlab/import_export/project_creator.rb @@ -0,0 +1,24 @@ +module Gitlab + module ImportExport + class ProjectCreator + + def initialize(namespace_id, current_user, file, project_path) + @namespace_id = namespace_id + @current_user = current_user + @file = file + @project_path = project_path + end + + def execute + ::Projects::CreateService.new( + @current_user, + name: @project_path, + path: @project_path, + namespace_id: @namespace_id, + import_type: "gitlab_project", + import_source: @file + ).execute + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb new file mode 100644 index 00000000000..dd71b92c522 --- /dev/null +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -0,0 +1,105 @@ +module Gitlab + module ImportExport + class ProjectTreeRestorer + + def initialize(user:, shared:, project:) + @path = File.join(shared.export_path, 'project.json') + @user = user + @shared = shared + @project = project + end + + def restore + json = IO.read(@path) + @tree_hash = ActiveSupport::JSON.decode(json) + @project_members = @tree_hash.delete('project_members') + create_relations + rescue => e + @shared.error(e) + false + end + + def restored_project + @restored_project ||= restore_project + end + + private + + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, + user: @user, + project: restored_project) + end + + # Loops through the tree of models defined in import_export.yml and + # finds them in the imported JSON so they can be instantiated and saved + # in the DB. The structure and relationships between models are guessed from + # the configuration yaml file too. + # Finally, it updates each attribute in the newly imported project. + def create_relations + saved = [] + default_relation_list.each do |relation| + next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present? + + create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash) + + relation_key = relation.is_a?(Hash) ? relation.keys.first : relation + relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s]) + saved << restored_project.update_attribute(relation_key, relation_hash) + end + saved.all? + end + + def default_relation_list + Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model| + model.is_a?(Hash) && model[:project_members] + end + end + + def restore_project + return @project unless @tree_hash + + project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } + @project.update(project_params) + @project + end + + # Given a relation hash containing one or more models and its relationships, + # loops through each model and each object from a model type and + # and assigns its correspondent attributes hash from +tree_hash+ + # Example: + # +relation_key+ issues, loops through the list of *issues* and for each individual + # issue, finds any subrelations such as notes, creates them and assign them back to the hash + def create_sub_relations(relation, tree_hash) + relation_key = relation.keys.first.to_s + tree_hash[relation_key].each do |relation_item| + relation.values.flatten.each do |sub_relation| + relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) + relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? + end + end + end + + def assign_relation_hash(relation_item, sub_relation) + if sub_relation.is_a?(Hash) + relation_hash = relation_item[sub_relation.keys.first.to_s] + sub_relation = sub_relation.keys.first + else + relation_hash = relation_item[sub_relation.to_s] + end + [relation_hash, sub_relation] + end + + def create_relation(relation, relation_hash_list) + relation_array = [relation_hash_list].flatten.map do |relation_hash| + Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, + relation_hash: relation_hash.merge('project_id' => restored_project.id), + members_mapper: members_mapper, + user: @user) + end + + relation_hash_list.is_a?(Array) ? relation_array : relation_array.first + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb new file mode 100644 index 00000000000..9153088e966 --- /dev/null +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -0,0 +1,29 @@ +module Gitlab + module ImportExport + class ProjectTreeSaver + attr_reader :full_path + + def initialize(project:, shared:) + @project = project + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) + end + + def save + FileUtils.mkdir_p(@shared.export_path) + + File.write(full_path, project_json_tree) + true + rescue => e + @shared.error(e) + false + end + + private + + def project_json_tree + @project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree) + end + end + end +end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb new file mode 100644 index 00000000000..19defd8f03a --- /dev/null +++ b/lib/gitlab/import_export/reader.rb @@ -0,0 +1,117 @@ +module Gitlab + module ImportExport + class Reader + + attr_reader :tree + + def initialize(shared:) + @shared = shared + config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys + @tree = config_hash[:project_tree] + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], + excluded_attributes: config_hash[:excluded_attributes], + methods: config_hash[:methods]) + end + + # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # for outputting a project in JSON format, including its relations and sub relations. + def project_tree + @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) + rescue => e + @shared.error(e) + false + end + + private + + # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # + # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file + def build_hash(model_list) + model_list.map do |model_objects| + if model_objects.is_a?(Hash) + build_json_config_hash(model_objects) + else + @attributes_finder.find(model_objects) + end + end + end + + # Called when the model is actually a hash containing other relations (more models) + # Returns the config in the right format for calling +to_json+ + # +model_object_hash+ - A model relationship such as: + # {:merge_requests=>[:merge_request_diff, :notes]} + def build_json_config_hash(model_object_hash) + @json_config_hash = {} + + model_object_hash.values.flatten.each do |model_object| + current_key = model_object_hash.keys.first + + @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } + + handle_model_object(current_key, model_object) + process_sub_model(current_key, model_object) if model_object.is_a?(Hash) + end + @json_config_hash + end + + + # If the model is a hash, process the sub_models, which could also be hashes + # If there is a list, add to an existing array, otherwise use hash syntax + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash + def process_sub_model(current_key, model_object) + sub_model_json = build_json_config_hash(model_object).dup + @json_config_hash.slice!(current_key) + + if @json_config_hash[current_key] && @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] << sub_model_json + else + @json_config_hash[current_key] = { include: sub_model_json } + end + end + + # Creates or adds to an existing hash an individual model or list + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash + def handle_model_object(current_key, model_object) + if @json_config_hash[current_key] + add_model_value(current_key, model_object) + else + create_model_value(current_key, model_object) + end + end + + # Constructs a new hash that will hold the configuration for that particular object + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + def create_model_value(current_key, value) + parsed_hash = { include: value } + + @attributes_finder.parse(value) do |hash| + parsed_hash = { include: hash_or_merge(value, hash) } + end + @json_config_hash[current_key] = parsed_hash + end + + # Adds new model configuration to an existing hash with key +current_key+ + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + def add_model_value(current_key, value) + @attributes_finder.parse(value) { |hash| value = { value => hash } } + old_values = @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten + end + + # Construct a new hash or merge with an existing one a model configuration + # This is to fulfil +to_json+ requirements. + # +value+ existing model to be included in the hash + # +hash+ hash containing configuration generated mainly from +@attributes_finder+ + def hash_or_merge(value, hash) + value.is_a?(Hash) ? value.merge(hash) : { value => hash } + end + end + end +end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb new file mode 100644 index 00000000000..b872780f20a --- /dev/null +++ b/lib/gitlab/import_export/relation_factory.rb @@ -0,0 +1,128 @@ +module Gitlab + module ImportExport + class RelationFactory + + OVERRIDES = { snippets: :project_snippets, + pipelines: 'Ci::Pipeline', + statuses: 'commit_status', + variables: 'Ci::Variable', + triggers: 'Ci::Trigger', + builds: 'Ci::Build', + hooks: 'ProjectHook' }.freeze + + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + + def self.create(*args) + new(*args).create + end + + def initialize(relation_sym:, relation_hash:, members_mapper:, user:) + @relation_name = OVERRIDES[relation_sym] || relation_sym + @relation_hash = relation_hash.except('id', 'noteable_id') + @members_mapper = members_mapper + @user = user + end + + # Creates an object from an actual model with name "relation_sym" with params from + # the relation_hash, updating references with new object IDs, mapping users using + # the "members_mapper" object, also updating notes if required. + def create + set_note_author if @relation_name == :notes + update_user_references + update_project_references + reset_ci_tokens if @relation_name == 'Ci::Trigger' + + generate_imported_object + end + + private + + def update_user_references + USER_REFERENCES.each do |reference| + if @relation_hash[reference] + @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] + end + end + end + + # Sets the author for a note. If the user importing the project + # has admin access, an actual mapping with new project members + # will be used. Otherwise, a note stating the original author name + # is left. + def set_note_author + old_author_id = @relation_hash['author_id'] + + # Users with admin access can map users + @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id + + author = @relation_hash.delete('author') + + update_note_for_missing_author(author['name']) if missing_author?(old_author_id) + end + + def missing_author?(old_author_id) + !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id) + end + + def missing_author_note(updated_at, author_name) + timestamp = updated_at.split('.').first + "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" + end + + def generate_imported_object + if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes + trace = @relation_hash.delete('trace') + imported_object do |object| + object.trace = trace + object.commit_id = nil + end + else + imported_object + end + end + + def update_project_references + project_id = @relation_hash.delete('project_id') + + # project_id may not be part of the export, but we always need to populate it if required. + @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id') + @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id'] + @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] + @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id'] + + # If source and target are the same, populate them with the new project ID. + if @relation_hash['source_project_id'] && @relation_hash['target_project_id'] && + @relation_hash['target_project_id'] == @relation_hash['source_project_id'] + @relation_hash['source_project_id'] = project_id + end + end + + def reset_ci_tokens + return unless Gitlab::ImportExport.reset_tokens? + + # If we import/export a project to the same instance, tokens will have to be reset. + @relation_hash['token'] = nil + end + + def relation_class + @relation_class ||= @relation_name.to_s.classify.constantize + end + + def imported_object + imported_object = relation_class.new(@relation_hash) + yield(imported_object) if block_given? + imported_object.importing = true if imported_object.respond_to?(:importing) + imported_object + end + + def update_note_for_missing_author(author_name) + @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? + @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name) + end + + def admin_user? + @user.is_admin? + end + end + end +end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb new file mode 100644 index 00000000000..546dae4d122 --- /dev/null +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -0,0 +1,39 @@ +module Gitlab + module ImportExport + class RepoRestorer + include Gitlab::ImportExport::CommandLineUtil + + def initialize(project:, shared:, path_to_bundle:, wiki: false) + @project = project + @path_to_bundle = path_to_bundle + @shared = shared + @wiki = wiki + end + + def restore + return wiki? unless File.exist?(@path_to_bundle) + + FileUtils.mkdir_p(path_to_repo) + + git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) + rescue => e + @shared.error(e) + false + end + + private + + def repos_path + Gitlab.config.gitlab_shell.repos_path + end + + def path_to_repo + @project.repository.path_to_repo + end + + def wiki? + @wiki + end + end + end +end diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb new file mode 100644 index 00000000000..cce43fe994b --- /dev/null +++ b/lib/gitlab/import_export/repo_saver.rb @@ -0,0 +1,35 @@ +module Gitlab + module ImportExport + class RepoSaver + include Gitlab::ImportExport::CommandLineUtil + + attr_reader :full_path + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return false if @project.empty_repo? + + @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) + bundle_to_disk + end + + private + + def bundle_to_disk + FileUtils.mkdir_p(@shared.export_path) + git_bundle(repo_path: path_to_repo, bundle_path: @full_path) + rescue => e + @shared.error(e) + false + end + + def path_to_repo + @project.repository.path_to_repo + end + end + end +end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb new file mode 100644 index 00000000000..f38229c6c59 --- /dev/null +++ b/lib/gitlab/import_export/saver.rb @@ -0,0 +1,42 @@ +module Gitlab + module ImportExport + class Saver + include Gitlab::ImportExport::CommandLineUtil + + def self.save(*args) + new(*args).save + end + + def initialize(shared:) + @shared = shared + end + + def save + if compress_and_save + remove_export_path + Rails.logger.info("Saved project export #{archive_file}") + archive_file + else + false + end + rescue => e + @shared.error(e) + false + end + + private + + def compress_and_save + tar_czf(archive: archive_file, dir: @shared.export_path) + end + + def remove_export_path + FileUtils.rm_rf(@shared.export_path) + end + + def archive_file + @archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz") + end + end + end +end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb new file mode 100644 index 00000000000..6aff05b886a --- /dev/null +++ b/lib/gitlab/import_export/shared.rb @@ -0,0 +1,30 @@ +module Gitlab + module ImportExport + class Shared + + attr_reader :errors, :opts + + def initialize(opts) + @opts = opts + @errors = [] + end + + def export_path + @export_path ||= Gitlab::ImportExport.export_path(relative_path: opts[:relative_path]) + end + + def error(error) + error_out(error.message, caller[0].dup) + @errors << error.message + # Debug: + Rails.logger.error(error.backtrace) + end + + private + + def error_out(message, caller) + Rails.logger.error("Import/Export error raised on #{caller}: #{message}") + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb new file mode 100644 index 00000000000..df19354b76e --- /dev/null +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -0,0 +1,14 @@ +module Gitlab + module ImportExport + class UploadsRestorer < UploadsSaver + def restore + return true unless File.directory?(uploads_export_path) + + copy_files(uploads_export_path, uploads_path) + rescue => e + @shared.error(e) + false + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb new file mode 100644 index 00000000000..7292e9d9712 --- /dev/null +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -0,0 +1,36 @@ +module Gitlab + module ImportExport + class UploadsSaver + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return true unless File.directory?(uploads_path) + + copy_files(uploads_path, uploads_export_path) + rescue => e + @shared.error(e) + false + end + + private + + def copy_files(source, destination) + FileUtils.mkdir_p(destination) + FileUtils.copy_entry(source, destination) + true + end + + def uploads_export_path + File.join(@shared.export_path, 'uploads') + end + + def uploads_path + File.join(Rails.root.join('public/uploads'), @project.path_with_namespace) + end + end + end +end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb new file mode 100644 index 00000000000..cf5c62c5e3c --- /dev/null +++ b/lib/gitlab/import_export/version_checker.rb @@ -0,0 +1,36 @@ +module Gitlab + module ImportExport + class VersionChecker + + def self.check!(*args) + new(*args).check! + end + + def initialize(shared:) + @shared = shared + end + + def check! + version = File.open(version_file, &:readline) + verify_version!(version) + rescue => e + @shared.error(e) + false + end + + private + + def version_file + File.join(@shared.export_path, Gitlab::ImportExport.version_filename) + end + + def verify_version!(version) + if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) + raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") + else + true + end + end + end + end +end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb new file mode 100644 index 00000000000..f7f73dc9343 --- /dev/null +++ b/lib/gitlab/import_export/version_saver.rb @@ -0,0 +1,25 @@ +module Gitlab + module ImportExport + class VersionSaver + + def initialize(shared:) + @shared = shared + end + + def save + FileUtils.mkdir_p(@shared.export_path) + + File.write(version_file, Gitlab::ImportExport.version, mode: 'w') + rescue => e + @shared.error(e) + false + end + + private + + def version_file + File.join(@shared.export_path, Gitlab::ImportExport.version_filename) + end + end + end +end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb new file mode 100644 index 00000000000..1eedae39f8a --- /dev/null +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -0,0 +1,33 @@ +module Gitlab + module ImportExport + class WikiRepoSaver < RepoSaver + def save + @wiki = ProjectWiki.new(@project) + return true unless wiki_repository_exists? # it's okay to have no Wiki + bundle_to_disk(File.join(@shared.export_path, project_filename)) + end + + def bundle_to_disk(full_path) + FileUtils.mkdir_p(@shared.export_path) + git_bundle(repo_path: path_to_repo, bundle_path: full_path) + rescue => e + @shared.error(e) + false + end + + private + + def project_filename + "project.wiki.bundle" + end + + def path_to_repo + @wiki.repository.path_to_repo + end + + def wiki_repository_exists? + File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? + end + end + end +end |