diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-26 18:09:24 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-26 18:09:24 +0000 |
| commit | 619d0b6922a6cf95d291fbbf5fa3d09e772a1ea8 (patch) | |
| tree | fb8f8e036cec1b32166206bb5102af6c5dca8cfe /lib | |
| parent | 17ab40ca089e1aef61a83f77ab6df62a72f6ce06 (diff) | |
| download | gitlab-ce-619d0b6922a6cf95d291fbbf5fa3d09e772a1ea8.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
40 files changed, 1255 insertions, 1232 deletions
diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 382bbeb66de..577a6e890d7 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -49,8 +49,8 @@ module API result = access_checker.check(params[:action], params[:changes]) @project ||= access_checker.project result - rescue Gitlab::GitAccess::UnauthorizedError => e - return response_with_status(code: 401, success: false, message: e.message) + rescue Gitlab::GitAccess::ForbiddenError => e + return response_with_status(code: 403, success: false, message: e.message) rescue Gitlab::GitAccess::TimeoutError => e return response_with_status(code: 503, success: false, message: e.message) rescue Gitlab::GitAccess::NotFoundError => e diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index 4ddc1c718c7..7be0ef05a49 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -28,7 +28,7 @@ module Gitlab logger.log_timed(LOG_MESSAGES[:delete_default_branch_check]) do if deletion? && branch_name == project.default_branch - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_default_branch] end end @@ -42,7 +42,7 @@ module Gitlab return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks if forced_push? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:force_push_protected_branch] end end @@ -62,15 +62,15 @@ module Gitlab break if user_access.can_push_to_branch?(branch_name) unless user_access.can_merge_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_branch] end unless safe_commit_for_new_protected_branch? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:invalid_commit_create_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:invalid_commit_create_protected_branch] end unless updated_from_web? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_create_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_web_create_protected_branch] end end end @@ -78,11 +78,11 @@ module Gitlab def protected_branch_deletion_checks logger.log_timed(LOG_MESSAGES[:protected_branch_deletion_checks]) do unless user_access.can_delete_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_master_delete_protected_branch] end unless updated_from_web? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_web_delete_protected_branch] end end end @@ -91,11 +91,11 @@ module Gitlab logger.log_timed(LOG_MESSAGES[:protected_branch_push_checks]) do if matching_merge_request? unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:merge_protected_branch] end else unless user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message + raise GitAccess::ForbiddenError, push_to_protected_branch_rejected_message end end end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index 5de71addd5f..0eb2b4c79ef 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -46,7 +46,7 @@ module Gitlab def validate_diff(diff) validations_for_diff.each do |validation| if error = validation.call(diff) - raise ::Gitlab::GitAccess::UnauthorizedError, error + raise ::Gitlab::GitAccess::ForbiddenError, error end end end @@ -77,7 +77,7 @@ module Gitlab logger.log_timed(LOG_MESSAGES[__method__]) do path_validations.each do |validation| if error = validation.call(file_paths) - raise ::Gitlab::GitAccess::UnauthorizedError, error + raise ::Gitlab::GitAccess::ForbiddenError, error end end end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index 7b013567a03..f81c215d847 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -15,7 +15,7 @@ module Gitlab lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) if lfs_check.objects_missing? - raise GitAccess::UnauthorizedError, ERROR_MESSAGE + raise GitAccess::ForbiddenError, ERROR_MESSAGE end end end diff --git a/lib/gitlab/checks/push_check.rb b/lib/gitlab/checks/push_check.rb index 91f8d0bdbc8..7cc5bc56cbb 100644 --- a/lib/gitlab/checks/push_check.rb +++ b/lib/gitlab/checks/push_check.rb @@ -6,7 +6,7 @@ module Gitlab def validate! logger.log_timed("Checking if you are allowed to push...") do unless can_push? - raise GitAccess::UnauthorizedError, GitAccess::ERROR_MESSAGES[:push_code] + raise GitAccess::ForbiddenError, GitAccess::ERROR_MESSAGES[:push_code] end end end diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb index ced0612a7a3..a47e55cb160 100644 --- a/lib/gitlab/checks/tag_check.rb +++ b/lib/gitlab/checks/tag_check.rb @@ -20,7 +20,7 @@ module Gitlab logger.log_timed(LOG_MESSAGES[:tag_checks]) do if tag_exists? && user_access.cannot_do_action?(:admin_tag) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:change_existing_tags] end end @@ -33,11 +33,11 @@ module Gitlab logger.log_timed(LOG_MESSAGES[__method__]) do return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? + raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:update_protected_tag]) if update? + raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? unless user_access.can_create_tag?(tag_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_tag] end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 6bfe744a5cd..29324381cb5 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -322,6 +322,7 @@ module Gitlab limit: 10, offset: 0, path: nil, + author: nil, follow: false, skip_merges: false, after: nil, diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 906350e57c5..d6c87b858a8 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -6,7 +6,7 @@ module Gitlab class GitAccess include Gitlab::Utils::StrongMemoize - UnauthorizedError = Class.new(StandardError) + ForbiddenError = Class.new(StandardError) NotFoundError = Class.new(StandardError) ProjectCreationError = Class.new(StandardError) TimeoutError = Class.new(StandardError) @@ -125,7 +125,7 @@ module Gitlab return unless actor.is_a?(Key) unless actor.valid? - raise UnauthorizedError, "Your SSH key #{actor.errors[:key].first}." + raise ForbiddenError, "Your SSH key #{actor.errors[:key].first}." end end @@ -133,7 +133,7 @@ module Gitlab return if request_from_ci_build? unless protocol_allowed? - raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed" + raise ForbiddenError, "Git access over #{protocol.upcase} is not allowed" end end @@ -148,7 +148,7 @@ module Gitlab unless user_access.allowed? message = Gitlab::Auth::UserAccessDeniedReason.new(user).rejection_message - raise UnauthorizedError, message + raise ForbiddenError, message end end @@ -156,11 +156,11 @@ module Gitlab case cmd when *DOWNLOAD_COMMANDS unless authentication_abilities.include?(:download_code) || authentication_abilities.include?(:build_download_code) - raise UnauthorizedError, ERROR_MESSAGES[:auth_download] + raise ForbiddenError, ERROR_MESSAGES[:auth_download] end when *PUSH_COMMANDS unless authentication_abilities.include?(:push_code) - raise UnauthorizedError, ERROR_MESSAGES[:auth_upload] + raise ForbiddenError, ERROR_MESSAGES[:auth_upload] end end end @@ -189,19 +189,19 @@ module Gitlab def check_upload_pack_disabled! if http? && upload_pack_disabled_over_http? - raise UnauthorizedError, ERROR_MESSAGES[:upload_pack_disabled_over_http] + raise ForbiddenError, ERROR_MESSAGES[:upload_pack_disabled_over_http] end end def check_receive_pack_disabled! if http? && receive_pack_disabled_over_http? - raise UnauthorizedError, ERROR_MESSAGES[:receive_pack_disabled_over_http] + raise ForbiddenError, ERROR_MESSAGES[:receive_pack_disabled_over_http] end end def check_command_existence!(cmd) unless ALL_COMMANDS.include?(cmd) - raise UnauthorizedError, ERROR_MESSAGES[:command_not_allowed] + raise ForbiddenError, ERROR_MESSAGES[:command_not_allowed] end end @@ -209,7 +209,7 @@ module Gitlab return unless receive_pack?(cmd) if Gitlab::Database.read_only? - raise UnauthorizedError, push_to_read_only_message + raise ForbiddenError, push_to_read_only_message end end @@ -253,23 +253,23 @@ module Gitlab guest_can_download_code? unless passed - raise UnauthorizedError, ERROR_MESSAGES[:download] + raise ForbiddenError, ERROR_MESSAGES[:download] end end def check_push_access! if project.repository_read_only? - raise UnauthorizedError, ERROR_MESSAGES[:read_only] + raise ForbiddenError, ERROR_MESSAGES[:read_only] end if deploy_key? unless deploy_key.can_push_to?(project) - raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] + raise ForbiddenError, ERROR_MESSAGES[:deploy_key_upload] end elsif user # User access is verified in check_change_access! else - raise UnauthorizedError, ERROR_MESSAGES[:upload] + raise ForbiddenError, ERROR_MESSAGES[:upload] end check_change_access! @@ -284,7 +284,7 @@ module Gitlab project.any_branch_allows_collaboration?(user_access.user) unless can_push - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] + raise ForbiddenError, ERROR_MESSAGES[:push_code] end else # If there are worktrees with a HEAD pointing to a non-existent object, diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 3d0db753f6e..aad46937c32 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -19,11 +19,11 @@ module Gitlab def check_change_access! unless user_access.can_do_action?(:create_wiki) - raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] + raise ForbiddenError, ERROR_MESSAGES[:write_to_wiki] end if Gitlab::Database.read_only? - raise UnauthorizedError, push_to_read_only_message + raise ForbiddenError, push_to_read_only_message end true diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index ac22f5bf419..1f914dc95d1 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -324,7 +324,8 @@ module Gitlab request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] request.revision = encode_binary(options[:ref]) if options[:ref] - request.order = options[:order].upcase.sub('DEFAULT', 'NONE') if options[:order].present? + request.author = encode_binary(options[:author]) if options[:author] + request.order = options[:order].upcase.sub('DEFAULT', 'NONE') if options[:order].present? request.paths = encode_repeated(Array(options[:path])) if options[:path].present? diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 8ce6549c0c7..1033e6c4e05 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -43,7 +43,7 @@ module Gitlab end def config_file - Rails.root.join('lib/gitlab/import_export/import_export.yml') + Rails.root.join('lib/gitlab/import_export/project/import_export.yml') end def version_filename @@ -77,7 +77,7 @@ module Gitlab end def group_config_file - Rails.root.join('lib/gitlab/import_export/group_import_export.yml') + Rails.root.join('lib/gitlab/import_export/group/import_export.yml') end end end diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index d1c20dff799..3bfc059dcd3 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -4,8 +4,8 @@ module Gitlab module ImportExport class AttributeCleaner ALLOWED_REFERENCES = [ - *ProjectRelationFactory::PROJECT_REFERENCES, - *ProjectRelationFactory::USER_REFERENCES, + *Gitlab::ImportExport::Project::RelationFactory::PROJECT_REFERENCES, + *Gitlab::ImportExport::Project::RelationFactory::USER_REFERENCES, 'group_id', 'commit_id', 'discussion_id', diff --git a/lib/gitlab/import_export/base/object_builder.rb b/lib/gitlab/import_export/base/object_builder.rb new file mode 100644 index 00000000000..109d2e233a5 --- /dev/null +++ b/lib/gitlab/import_export/base/object_builder.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Base + # Base class for Group & Project Object Builders. + # This class is not intended to be used on its own but + # rather inherited from. + # + # Cache keeps 1000 entries at most, 1000 is chosen based on: + # - one cache entry uses around 0.5K memory, 1000 items uses around 500K. + # (leave some buffer it should be less than 1M). It is afforable cost for project import. + # - for projects in Gitlab.com, it seems 1000 entries for labels/milestones is enough. + # For example, gitlab has ~970 labels and 26 milestones. + LRU_CACHE_SIZE = 1000 + + class ObjectBuilder + def self.build(*args) + new(*args).find + end + + def initialize(klass, attributes) + @klass = klass.ancestors.include?(Label) ? Label : klass + @attributes = attributes + + if Gitlab::SafeRequestStore.active? + @lru_cache = cache_from_request_store + @cache_key = [klass, attributes] + end + end + + def find + find_with_cache do + find_object || klass.create(prepare_attributes) + end + end + + protected + + def where_clauses + raise NotImplementedError + end + + # attributes wrapped in a method to be + # adjusted in sub-class if needed + def prepare_attributes + attributes + end + + private + + attr_reader :klass, :attributes, :lru_cache, :cache_key + + def find_with_cache + return yield unless lru_cache && cache_key + + lru_cache[cache_key] ||= yield + end + + def cache_from_request_store + Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE) + end + + def find_object + klass.where(where_clause).first + end + + def where_clause + where_clauses.reduce(:and) + end + + def table + @table ||= klass.arel_table + end + + # Returns Arel clause: + # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"` + # from the given Hash of attributes. + def attrs_to_arel(attrs) + attrs.map do |key, value| + table[key].eq(value) + end.reduce(:and) + end + + # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'` + # if attributes has 'title key, otherwise `nil`. + def where_clause_for_title + attrs_to_arel(attributes.slice('title')) + end + + # Returns Arel clause `"{table_name}"."description" = '{attributes['description']}'` + # if attributes has 'description key, otherwise `nil`. + def where_clause_for_description + attrs_to_arel(attributes.slice('description')) + end + + # Returns Arel clause `"{table_name}"."created_at" = '{attributes['created_at']}'` + # if attributes has 'created_at key, otherwise `nil`. + def where_clause_for_created_at + attrs_to_arel(attributes.slice('created_at')) + end + end + end + end +end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb new file mode 100644 index 00000000000..688627d1f2f --- /dev/null +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -0,0 +1,308 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Base + class RelationFactory + include Gitlab::Utils::StrongMemoize + + IMPORTED_OBJECT_MAX_RETRIES = 5.freeze + + OVERRIDES = {}.freeze + EXISTING_OBJECT_RELATIONS = %i[].freeze + + # This represents all relations that have unique key on `project_id` or `group_id` + UNIQUE_RELATIONS = %i[].freeze + + USER_REFERENCES = %w[ + author_id + assignee_id + updated_by_id + merged_by_id + latest_closed_by_id + user_id + created_by_id + last_edited_by_id + merge_user_id + resolved_by_id + closed_by_id + owner_id + ].freeze + + TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze + + def self.create(*args) + new(*args).create + end + + def self.relation_class(relation_name) + # There are scenarios where the model is pluralized (e.g. + # MergeRequest::Metrics), and we don't want to force it to singular + # with #classify. + relation_name.to_s.classify.constantize + rescue NameError + relation_name.to_s.constantize + end + + def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: []) + @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym + @relation_hash = relation_hash.except('noteable_id') + @members_mapper = members_mapper + @object_builder = object_builder + @user = user + @importable = importable + @imported_object_retries = 0 + @relation_hash[importable_column_name] = @importable.id + + # Remove excluded keys from relation_hash + # We don't do this in the parsed_relation_hash because of the 'transformed attributes' + # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then, + # in the create method that attribute is renamed to diff. And because diff is an excluded key, + # if we clean the excluded keys in the parsed_relation_hash, it will be removed + # from the object attributes and the export will fail. + @relation_hash.except!(*excluded_keys) + end + + # Creates an object from an actual model with name "relation_sym" with params from + # the relation_hash, updating references with new object IDs, mapping users using + # the "members_mapper" object, also updating notes if required. + def create + return if invalid_relation? + + setup_base_models + setup_models + + generate_imported_object + end + + def self.overrides + self::OVERRIDES + end + + def self.existing_object_relations + self::EXISTING_OBJECT_RELATIONS + end + + private + + def invalid_relation? + false + end + + def setup_models + raise NotImplementedError + end + + def unique_relations + # define in sub-class if any + self.class::UNIQUE_RELATIONS + end + + def setup_base_models + update_user_references + remove_duplicate_assignees + reset_tokens! + remove_encrypted_attributes! + end + + def update_user_references + self.class::USER_REFERENCES.each do |reference| + if @relation_hash[reference] + @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] + end + end + end + + def remove_duplicate_assignees + return unless @relation_hash['issue_assignees'] + + # When an assignee did not exist in the members mapper, the importer is + # assigned. We only need to assign each user once. + @relation_hash['issue_assignees'].uniq!(&:user_id) + end + + def generate_imported_object + imported_object + end + + def reset_tokens! + return unless Gitlab::ImportExport.reset_tokens? && self.class::TOKEN_RESET_MODELS.include?(@relation_name) + + # If we import/export to the same instance, tokens will have to be reset. + # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across. + relation_class.attribute_names.select { |name| name.include?('token') }.each do |token| + @relation_hash[token] = nil + end + end + + def remove_encrypted_attributes! + return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any? + + relation_class.encrypted_attributes.each_key do |key| + @relation_hash[key.to_s] = nil + end + end + + def relation_class + @relation_class ||= self.class.relation_class(@relation_name) + end + + def importable_column_name + importable_class_name.concat('_id') + end + + def importable_class_name + @importable.class.to_s.downcase + end + + def imported_object + if existing_or_new_object.respond_to?(:importing) + existing_or_new_object.importing = true + end + + existing_or_new_object + rescue ActiveRecord::RecordNotUnique + # as the operation is not atomic, retry in the unlikely scenario an INSERT is + # performed on the same object between the SELECT and the INSERT + @imported_object_retries += 1 + retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES + end + + def parsed_relation_hash + @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, + relation_class: relation_class) + end + + def existing_or_new_object + # Only find existing records to avoid mapping tables such as milestones + # Otherwise always create the record, skipping the extra SELECT clause. + @existing_or_new_object ||= begin + if existing_object? + attribute_hash = attribute_hash_for(['events']) + + existing_object.assign_attributes(attribute_hash) if attribute_hash.any? + + existing_object + else + # Because of single-type inheritance, we need to be careful to use the `type` field + # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497 + inheritance_column = relation_class.try(:inheritance_column) + inheritance_attributes = parsed_relation_hash.slice(inheritance_column) + object = relation_class.new(inheritance_attributes) + object.assign_attributes(parsed_relation_hash) + object + end + end + end + + def attribute_hash_for(attributes) + attributes.each_with_object({}) do |hash, value| + hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value] + hash + end + end + + def existing_object + @existing_object ||= find_or_create_object! + end + + def unique_relation_object + unique_relation_object = relation_class.find_or_create_by(importable_column_name => @importable.id) + unique_relation_object.assign_attributes(parsed_relation_hash) + unique_relation_object + end + + def find_or_create_object! + return unique_relation_object if unique_relation? + + # Can't use IDs as validation exists calling `group` or `project` attributes + finder_hash = parsed_relation_hash.tap do |hash| + if relation_class.attribute_method?('group_id') && @importable.is_a?(::Project) + hash['group'] = @importable.group + end + + hash[importable_class_name] = @importable if relation_class.reflect_on_association(importable_class_name.to_sym) + hash.delete(importable_column_name) + end + + @object_builder.build(relation_class, finder_hash) + end + + def setup_note + set_note_author + # attachment is deprecated and note uploads are handled by Markdown uploader + @relation_hash['attachment'] = nil + end + + # Sets the author for a note. If the user importing the project + # has admin access, an actual mapping with new project members + # will be used. Otherwise, a note stating the original author name + # is left. + def set_note_author + old_author_id = @relation_hash['author_id'] + author = @relation_hash.delete('author') + + update_note_for_missing_author(author['name']) unless has_author?(old_author_id) + end + + def has_author?(old_author_id) + admin_user? && @members_mapper.include?(old_author_id) + end + + def missing_author_note(updated_at, author_name) + timestamp = updated_at.split('.').first + "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" + end + + def update_note_for_missing_author(author_name) + @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? + @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}" + end + + def admin_user? + @user.admin? + end + + def existing_object? + strong_memoize(:_existing_object) do + self.class.existing_object_relations.include?(@relation_name) || unique_relation? + end + end + + def unique_relation? + strong_memoize(:unique_relation) do + importable_foreign_key.present? && + (has_unique_index_on_importable_fk? || uses_importable_fk_as_primary_key?) + end + end + + def has_unique_index_on_importable_fk? + cache = cached_has_unique_index_on_importable_fk + table_name = relation_class.table_name + return cache[table_name] if cache.has_key?(table_name) + + index_exists = + ActiveRecord::Base.connection.index_exists?( + relation_class.table_name, + importable_foreign_key, + unique: true) + + cache[table_name] = index_exists + end + + # Avoid unnecessary DB requests + def cached_has_unique_index_on_importable_fk + Thread.current[:cached_has_unique_index_on_importable_fk] ||= {} + end + + def uses_importable_fk_as_primary_key? + relation_class.primary_key == importable_foreign_key + end + + def importable_foreign_key + relation_class.reflect_on_association(importable_class_name.to_sym)&.foreign_key + end + end + end + end +end diff --git a/lib/gitlab/import_export/base_object_builder.rb b/lib/gitlab/import_export/base_object_builder.rb deleted file mode 100644 index ec66b7a7a4f..00000000000 --- a/lib/gitlab/import_export/base_object_builder.rb +++ /dev/null @@ -1,103 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - # Base class for Group & Project Object Builders. - # This class is not intended to be used on its own but - # rather inherited from. - # - # Cache keeps 1000 entries at most, 1000 is chosen based on: - # - one cache entry uses around 0.5K memory, 1000 items uses around 500K. - # (leave some buffer it should be less than 1M). It is afforable cost for project import. - # - for projects in Gitlab.com, it seems 1000 entries for labels/milestones is enough. - # For example, gitlab has ~970 labels and 26 milestones. - LRU_CACHE_SIZE = 1000 - - class BaseObjectBuilder - def self.build(*args) - new(*args).find - end - - def initialize(klass, attributes) - @klass = klass.ancestors.include?(Label) ? Label : klass - @attributes = attributes - - if Gitlab::SafeRequestStore.active? - @lru_cache = cache_from_request_store - @cache_key = [klass, attributes] - end - end - - def find - find_with_cache do - find_object || klass.create(prepare_attributes) - end - end - - protected - - def where_clauses - raise NotImplementedError - end - - # attributes wrapped in a method to be - # adjusted in sub-class if needed - def prepare_attributes - attributes - end - - private - - attr_reader :klass, :attributes, :lru_cache, :cache_key - - def find_with_cache - return yield unless lru_cache && cache_key - - lru_cache[cache_key] ||= yield - end - - def cache_from_request_store - Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE) - end - - def find_object - klass.where(where_clause).first - end - - def where_clause - where_clauses.reduce(:and) - end - - def table - @table ||= klass.arel_table - end - - # Returns Arel clause: - # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"` - # from the given Hash of attributes. - def attrs_to_arel(attrs) - attrs.map do |key, value| - table[key].eq(value) - end.reduce(:and) - end - - # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'` - # if attributes has 'title key, otherwise `nil`. - def where_clause_for_title - attrs_to_arel(attributes.slice('title')) - end - - # Returns Arel clause `"{table_name}"."description" = '{attributes['description']}'` - # if attributes has 'description key, otherwise `nil`. - def where_clause_for_description - attrs_to_arel(attributes.slice('description')) - end - - # Returns Arel clause `"{table_name}"."created_at" = '{attributes['created_at']}'` - # if attributes has 'created_at key, otherwise `nil`. - def where_clause_for_created_at - attrs_to_arel(attributes.slice('created_at')) - end - end - end -end diff --git a/lib/gitlab/import_export/base_relation_factory.rb b/lib/gitlab/import_export/base_relation_factory.rb deleted file mode 100644 index fcb516fb3a1..00000000000 --- a/lib/gitlab/import_export/base_relation_factory.rb +++ /dev/null @@ -1,306 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class BaseRelationFactory - include Gitlab::Utils::StrongMemoize - - IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - - OVERRIDES = {}.freeze - EXISTING_OBJECT_RELATIONS = %i[].freeze - - # This represents all relations that have unique key on `project_id` or `group_id` - UNIQUE_RELATIONS = %i[].freeze - - USER_REFERENCES = %w[ - author_id - assignee_id - updated_by_id - merged_by_id - latest_closed_by_id - user_id - created_by_id - last_edited_by_id - merge_user_id - resolved_by_id - closed_by_id - owner_id - ].freeze - - TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze - - def self.create(*args) - new(*args).create - end - - def self.relation_class(relation_name) - # There are scenarios where the model is pluralized (e.g. - # MergeRequest::Metrics), and we don't want to force it to singular - # with #classify. - relation_name.to_s.classify.constantize - rescue NameError - relation_name.to_s.constantize - end - - def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: []) - @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym - @relation_hash = relation_hash.except('noteable_id') - @members_mapper = members_mapper - @object_builder = object_builder - @user = user - @importable = importable - @imported_object_retries = 0 - @relation_hash[importable_column_name] = @importable.id - - # Remove excluded keys from relation_hash - # We don't do this in the parsed_relation_hash because of the 'transformed attributes' - # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then, - # in the create method that attribute is renamed to diff. And because diff is an excluded key, - # if we clean the excluded keys in the parsed_relation_hash, it will be removed - # from the object attributes and the export will fail. - @relation_hash.except!(*excluded_keys) - end - - # Creates an object from an actual model with name "relation_sym" with params from - # the relation_hash, updating references with new object IDs, mapping users using - # the "members_mapper" object, also updating notes if required. - def create - return if invalid_relation? - - setup_base_models - setup_models - - generate_imported_object - end - - def self.overrides - self::OVERRIDES - end - - def self.existing_object_relations - self::EXISTING_OBJECT_RELATIONS - end - - private - - def invalid_relation? - false - end - - def setup_models - raise NotImplementedError - end - - def unique_relations - # define in sub-class if any - self.class::UNIQUE_RELATIONS - end - - def setup_base_models - update_user_references - remove_duplicate_assignees - reset_tokens! - remove_encrypted_attributes! - end - - def update_user_references - self.class::USER_REFERENCES.each do |reference| - if @relation_hash[reference] - @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] - end - end - end - - def remove_duplicate_assignees - return unless @relation_hash['issue_assignees'] - - # When an assignee did not exist in the members mapper, the importer is - # assigned. We only need to assign each user once. - @relation_hash['issue_assignees'].uniq!(&:user_id) - end - - def generate_imported_object - imported_object - end - - def reset_tokens! - return unless Gitlab::ImportExport.reset_tokens? && self.class::TOKEN_RESET_MODELS.include?(@relation_name) - - # If we import/export to the same instance, tokens will have to be reset. - # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across. - relation_class.attribute_names.select { |name| name.include?('token') }.each do |token| - @relation_hash[token] = nil - end - end - - def remove_encrypted_attributes! - return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any? - - relation_class.encrypted_attributes.each_key do |key| - @relation_hash[key.to_s] = nil - end - end - - def relation_class - @relation_class ||= self.class.relation_class(@relation_name) - end - - def importable_column_name - importable_class_name.concat('_id') - end - - def importable_class_name - @importable.class.to_s.downcase - end - - def imported_object - if existing_or_new_object.respond_to?(:importing) - existing_or_new_object.importing = true - end - - existing_or_new_object - rescue ActiveRecord::RecordNotUnique - # as the operation is not atomic, retry in the unlikely scenario an INSERT is - # performed on the same object between the SELECT and the INSERT - @imported_object_retries += 1 - retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES - end - - def parsed_relation_hash - @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, - relation_class: relation_class) - end - - def existing_or_new_object - # Only find existing records to avoid mapping tables such as milestones - # Otherwise always create the record, skipping the extra SELECT clause. - @existing_or_new_object ||= begin - if existing_object? - attribute_hash = attribute_hash_for(['events']) - - existing_object.assign_attributes(attribute_hash) if attribute_hash.any? - - existing_object - else - # Because of single-type inheritance, we need to be careful to use the `type` field - # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497 - inheritance_column = relation_class.try(:inheritance_column) - inheritance_attributes = parsed_relation_hash.slice(inheritance_column) - object = relation_class.new(inheritance_attributes) - object.assign_attributes(parsed_relation_hash) - object - end - end - end - - def attribute_hash_for(attributes) - attributes.each_with_object({}) do |hash, value| - hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value] - hash - end - end - - def existing_object - @existing_object ||= find_or_create_object! - end - - def unique_relation_object - unique_relation_object = relation_class.find_or_create_by(importable_column_name => @importable.id) - unique_relation_object.assign_attributes(parsed_relation_hash) - unique_relation_object - end - - def find_or_create_object! - return unique_relation_object if unique_relation? - - # Can't use IDs as validation exists calling `group` or `project` attributes - finder_hash = parsed_relation_hash.tap do |hash| - if relation_class.attribute_method?('group_id') && @importable.is_a?(Project) - hash['group'] = @importable.group - end - - hash[importable_class_name] = @importable if relation_class.reflect_on_association(importable_class_name.to_sym) - hash.delete(importable_column_name) - end - - @object_builder.build(relation_class, finder_hash) - end - - def setup_note - set_note_author - # attachment is deprecated and note uploads are handled by Markdown uploader - @relation_hash['attachment'] = nil - end - - # Sets the author for a note. If the user importing the project - # has admin access, an actual mapping with new project members - # will be used. Otherwise, a note stating the original author name - # is left. - def set_note_author - old_author_id = @relation_hash['author_id'] - author = @relation_hash.delete('author') - - update_note_for_missing_author(author['name']) unless has_author?(old_author_id) - end - - def has_author?(old_author_id) - admin_user? && @members_mapper.include?(old_author_id) - end - - def missing_author_note(updated_at, author_name) - timestamp = updated_at.split('.').first - "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" - end - - def update_note_for_missing_author(author_name) - @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? - @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}" - end - - def admin_user? - @user.admin? - end - - def existing_object? - strong_memoize(:_existing_object) do - self.class.existing_object_relations.include?(@relation_name) || unique_relation? - end - end - - def unique_relation? - strong_memoize(:unique_relation) do - importable_foreign_key.present? && - (has_unique_index_on_importable_fk? || uses_importable_fk_as_primary_key?) - end - end - - def has_unique_index_on_importable_fk? - cache = cached_has_unique_index_on_importable_fk - table_name = relation_class.table_name - return cache[table_name] if cache.has_key?(table_name) - - index_exists = - ActiveRecord::Base.connection.index_exists?( - relation_class.table_name, - importable_foreign_key, - unique: true) - - cache[table_name] = index_exists - end - - # Avoid unnecessary DB requests - def cached_has_unique_index_on_importable_fk - Thread.current[:cached_has_unique_index_on_importable_fk] ||= {} - end - - def uses_importable_fk_as_primary_key? - relation_class.primary_key == importable_foreign_key - end - - def importable_foreign_key - relation_class.reflect_on_association(importable_class_name.to_sym)&.foreign_key - end - end - end -end diff --git a/lib/gitlab/import_export/group_import_export.yml b/lib/gitlab/import_export/group/import_export.yml index d4e0ff12373..d4e0ff12373 100644 --- a/lib/gitlab/import_export/group_import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml diff --git a/lib/gitlab/import_export/group/object_builder.rb b/lib/gitlab/import_export/group/object_builder.rb new file mode 100644 index 00000000000..e171a31348e --- /dev/null +++ b/lib/gitlab/import_export/group/object_builder.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + # Given a class, it finds or creates a new object at group level. + # + # Example: + # `Group::ObjectBuilder.build(Label, label_attributes)` + # finds or initializes a label with the given attributes. + class ObjectBuilder < Base::ObjectBuilder + def self.build(*args) + ::Group.transaction do + super + end + end + + def initialize(klass, attributes) + super + + @group = @attributes['group'] + + update_description + end + + private + + attr_reader :group + + # Convert description empty string to nil + # due to existing object being saved with description: nil + # Which makes object lookup to fail since nil != '' + def update_description + attributes['description'] = nil if attributes['description'] == '' + end + + def where_clauses + [ + where_clause_base, + where_clause_for_title, + where_clause_for_description, + where_clause_for_created_at + ].compact + end + + # Returns Arel clause `"{table_name}"."group_id" = {group.id}` + def where_clause_base + table[:group_id].in(group_and_ancestor_ids) + end + + def group_and_ancestor_ids + group.ancestors.map(&:id) << group.id + end + end + end + end +end diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb new file mode 100644 index 00000000000..91637161377 --- /dev/null +++ b/lib/gitlab/import_export/group/relation_factory.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + class RelationFactory < Base::RelationFactory + OVERRIDES = { + labels: :group_labels, + priorities: :label_priorities, + label: :group_label, + parent: :epic + }.freeze + + EXISTING_OBJECT_RELATIONS = %i[ + epic + epics + milestone + milestones + label + labels + group_label + group_labels + ].freeze + + private + + def setup_models + setup_note if @relation_name == :notes + + update_group_references + end + + def update_group_references + return unless self.class.existing_object_relations.include?(@relation_name) + return unless @relation_hash['group_id'] + + @relation_hash['group_id'] = @importable.id + end + end + end + end +end diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb new file mode 100644 index 00000000000..e6f49dcac7a --- /dev/null +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + class TreeRestorer + attr_reader :user + attr_reader :shared + attr_reader :group + + def initialize(user:, shared:, group:, group_hash:) + @path = File.join(shared.export_path, 'group.json') + @user = user + @shared = shared + @group = group + @group_hash = group_hash + end + + def restore + @tree_hash = @group_hash || read_tree_hash + @group_members = @tree_hash.delete('members') + @children = @tree_hash.delete('children') + + if members_mapper.map && restorer.restore + @children&.each do |group_hash| + group = create_group(group_hash: group_hash, parent_group: @group) + shared = Gitlab::ImportExport::Shared.new(group) + + self.class.new( + user: @user, + shared: shared, + group: group, + group_hash: group_hash + ).restore + end + end + + return false if @shared.errors.any? + + true + rescue => e + @shared.error(e) + false + end + + private + + def read_tree_hash + json = IO.read(@path) + ActiveSupport::JSON.decode(json) + rescue => e + @shared.logger.error( + group_id: @group.id, + group_name: @group.name, + message: "Import/Export error: #{e.message}" + ) + + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + + def restorer + @relation_tree_restorer ||= RelationTreeRestorer.new( + user: @user, + shared: @shared, + importable: @group, + tree_hash: @tree_hash.except('name', 'path'), + members_mapper: members_mapper, + object_builder: object_builder, + relation_factory: relation_factory, + reader: reader + ) + end + + def create_group(group_hash:, parent_group:) + group_params = { + name: group_hash['name'], + path: group_hash['path'], + parent_id: parent_group&.id, + visibility_level: sub_group_visibility_level(group_hash, parent_group) + } + + ::Groups::CreateService.new(@user, group_params).execute + end + + def sub_group_visibility_level(group_hash, parent_group) + original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE + + if parent_group && parent_group.visibility_level < original_visibility_level + Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level) + else + original_visibility_level + end + end + + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group) + end + + def relation_factory + Gitlab::ImportExport::Group::RelationFactory + end + + def object_builder + Gitlab::ImportExport::Group::ObjectBuilder + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new( + shared: @shared, + config: Gitlab::ImportExport::Config.new( + config: Gitlab::ImportExport.group_config_file + ).to_h + ) + end + end + end + end +end diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb new file mode 100644 index 00000000000..48f6925884b --- /dev/null +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + class TreeSaver + attr_reader :full_path, :shared + + def initialize(group:, current_user:, shared:, params: {}) + @params = params + @current_user = current_user + @shared = shared + @group = group + @full_path = File.join(@shared.export_path, ImportExport.group_filename) + end + + def save + group_tree = serialize(@group, reader.group_tree) + tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) + + true + rescue => e + @shared.error(e) + false + end + + private + + def serialize(group, relations_tree) + group_tree = tree_saver.serialize(group, relations_tree) + + group.children.each do |child| + group_tree['children'] ||= [] + group_tree['children'] << serialize(child, relations_tree) + end + + group_tree + rescue => e + @shared.error(e) + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new( + shared: @shared, + config: Gitlab::ImportExport::Config.new( + config: Gitlab::ImportExport.group_config_file + ).to_h + ) + end + + def tree_saver + @tree_saver ||= RelationTreeSaver.new + end + end + end + end +end diff --git a/lib/gitlab/import_export/group_object_builder.rb b/lib/gitlab/import_export/group_object_builder.rb deleted file mode 100644 index 9796bfa07d4..00000000000 --- a/lib/gitlab/import_export/group_object_builder.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - # Given a class, it finds or creates a new object at group level. - # - # Example: - # `GroupObjectBuilder.build(Label, label_attributes)` - # finds or initializes a label with the given attributes. - class GroupObjectBuilder < BaseObjectBuilder - def self.build(*args) - Group.transaction do - super - end - end - - def initialize(klass, attributes) - super - - @group = @attributes['group'] - - update_description - end - - private - - attr_reader :group - - # Convert description empty string to nil - # due to existing object being saved with description: nil - # Which makes object lookup to fail since nil != '' - def update_description - attributes['description'] = nil if attributes['description'] == '' - end - - def where_clauses - [ - where_clause_base, - where_clause_for_title, - where_clause_for_description, - where_clause_for_created_at - ].compact - end - - # Returns Arel clause `"{table_name}"."group_id" = {group.id}` - def where_clause_base - table[:group_id].in(group_and_ancestor_ids) - end - - def group_and_ancestor_ids - group.ancestors.map(&:id) << group.id - end - end - end -end diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb deleted file mode 100644 index 9e8f9d11393..00000000000 --- a/lib/gitlab/import_export/group_project_object_builder.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - # Given a class, it finds or creates a new object - # (initializes in the case of Label) at group or project level. - # If it does not exist in the group, it creates it at project level. - # - # Example: - # `GroupProjectObjectBuilder.build(Label, label_attributes)` - # finds or initializes a label with the given attributes. - # - # It also adds some logic around Group Labels/Milestones for edge cases. - class GroupProjectObjectBuilder < BaseObjectBuilder - def self.build(*args) - Project.transaction do - super - end - end - - def initialize(klass, attributes) - super - - @group = @attributes['group'] - @project = @attributes['project'] - end - - def find - return if epic? && group.nil? - - super - end - - private - - attr_reader :group, :project - - def where_clauses - [ - where_clause_base, - where_clause_for_title, - where_clause_for_klass - ].compact - end - - # Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present - # For example: merge_request has :target_project_id, and we are searching by :iid - # or, if group is present: - # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}` - def where_clause_base - [].tap do |clauses| - clauses << table[:project_id].eq(project.id) if project - clauses << table[:group_id].in(group.self_and_ancestors_ids) if group - end.reduce(:or) - end - - # Returns Arel clause for a particular model or `nil`. - def where_clause_for_klass - attrs_to_arel(attributes.slice('iid')) if merge_request? - end - - def prepare_attributes - attributes.dup.tap do |atts| - atts.delete('group') unless epic? - - if label? - atts['type'] = 'ProjectLabel' # Always create project labels - elsif milestone? - if atts['group_id'] # Transform new group milestones into project ones - atts['iid'] = nil - atts.delete('group_id') - else - claim_iid - end - end - - atts['importing'] = true if klass.ancestors.include?(Importable) - end - end - - def label? - klass == Label - end - - def milestone? - klass == Milestone - end - - def merge_request? - klass == MergeRequest - end - - def epic? - klass == Epic - end - - # If an existing group milestone used the IID - # claim the IID back and set the group milestone to use one available - # This is necessary to fix situations like the following: - # - Importing into a user namespace project with exported group milestones - # where the IID of the Group milestone could conflict with a project one. - def claim_iid - # The milestone has to be a group milestone, as it's the only case where - # we set the IID as the maximum. The rest of them are fixed. - milestone = project.milestones.find_by(iid: attributes['iid']) - - return unless milestone - - milestone.iid = nil - milestone.ensure_project_iid! - milestone.save! - end - end - end -end - -Gitlab::ImportExport::GroupProjectObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::GroupProjectObjectBuilder') diff --git a/lib/gitlab/import_export/group_relation_factory.rb b/lib/gitlab/import_export/group_relation_factory.rb deleted file mode 100644 index e3597af44d2..00000000000 --- a/lib/gitlab/import_export/group_relation_factory.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class GroupRelationFactory < BaseRelationFactory - OVERRIDES = { - labels: :group_labels, - priorities: :label_priorities, - label: :group_label, - parent: :epic - }.freeze - - EXISTING_OBJECT_RELATIONS = %i[ - epic - epics - milestone - milestones - label - labels - group_label - group_labels - ].freeze - - private - - def setup_models - setup_note if @relation_name == :notes - - update_group_references - end - - def update_group_references - return unless self.class.existing_object_relations.include?(@relation_name) - return unless @relation_hash['group_id'] - - @relation_hash['group_id'] = @importable.id - end - end - end -end diff --git a/lib/gitlab/import_export/group_tree_restorer.rb b/lib/gitlab/import_export/group_tree_restorer.rb deleted file mode 100644 index 2f42843ed6c..00000000000 --- a/lib/gitlab/import_export/group_tree_restorer.rb +++ /dev/null @@ -1,116 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class GroupTreeRestorer - attr_reader :user - attr_reader :shared - attr_reader :group - - def initialize(user:, shared:, group:, group_hash:) - @path = File.join(shared.export_path, 'group.json') - @user = user - @shared = shared - @group = group - @group_hash = group_hash - end - - def restore - @tree_hash = @group_hash || read_tree_hash - @group_members = @tree_hash.delete('members') - @children = @tree_hash.delete('children') - - if members_mapper.map && restorer.restore - @children&.each do |group_hash| - group = create_group(group_hash: group_hash, parent_group: @group) - shared = Gitlab::ImportExport::Shared.new(group) - - self.class.new( - user: @user, - shared: shared, - group: group, - group_hash: group_hash - ).restore - end - end - - return false if @shared.errors.any? - - true - rescue => e - @shared.error(e) - false - end - - private - - def read_tree_hash - json = IO.read(@path) - ActiveSupport::JSON.decode(json) - rescue => e - @shared.logger.error( - group_id: @group.id, - group_name: @group.name, - message: "Import/Export error: #{e.message}" - ) - - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') - end - - def restorer - @relation_tree_restorer ||= RelationTreeRestorer.new( - user: @user, - shared: @shared, - importable: @group, - tree_hash: @tree_hash.except('name', 'path'), - members_mapper: members_mapper, - object_builder: object_builder, - relation_factory: relation_factory, - reader: reader - ) - end - - def create_group(group_hash:, parent_group:) - group_params = { - name: group_hash['name'], - path: group_hash['path'], - parent_id: parent_group&.id, - visibility_level: sub_group_visibility_level(group_hash, parent_group) - } - - ::Groups::CreateService.new(@user, group_params).execute - end - - def sub_group_visibility_level(group_hash, parent_group) - original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE - - if parent_group && parent_group.visibility_level < original_visibility_level - Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level) - else - original_visibility_level - end - end - - def members_mapper - @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group) - end - - def relation_factory - Gitlab::ImportExport::GroupRelationFactory - end - - def object_builder - Gitlab::ImportExport::GroupObjectBuilder - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new( - shared: @shared, - config: Gitlab::ImportExport::Config.new( - config: Gitlab::ImportExport.group_config_file - ).to_h - ) - end - end - end -end diff --git a/lib/gitlab/import_export/group_tree_saver.rb b/lib/gitlab/import_export/group_tree_saver.rb deleted file mode 100644 index 2effcd01e30..00000000000 --- a/lib/gitlab/import_export/group_tree_saver.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class GroupTreeSaver - attr_reader :full_path, :shared - - def initialize(group:, current_user:, shared:, params: {}) - @params = params - @current_user = current_user - @shared = shared - @group = group - @full_path = File.join(@shared.export_path, ImportExport.group_filename) - end - - def save - group_tree = serialize(@group, reader.group_tree) - tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) - - true - rescue => e - @shared.error(e) - false - end - - private - - def serialize(group, relations_tree) - group_tree = tree_saver.serialize(group, relations_tree) - - group.children.each do |child| - group_tree['children'] ||= [] - group_tree['children'] << serialize(child, relations_tree) - end - - group_tree - rescue => e - @shared.error(e) - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new( - shared: @shared, - config: Gitlab::ImportExport::Config.new( - config: Gitlab::ImportExport.group_config_file - ).to_h - ) - end - - def tree_saver - @tree_saver ||= RelationTreeSaver.new - end - end - end -end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index a6463ed678c..4eeecc14067 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -49,7 +49,7 @@ module Gitlab end def project_tree - @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: current_user, + @project_tree ||= Gitlab::ImportExport::Project::TreeRestorer.new(user: current_user, shared: shared, project: project) end @@ -125,7 +125,7 @@ module Gitlab def project_to_overwrite strong_memoize(:project_to_overwrite) do - Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}") + ::Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}") end end end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index e7eae0a8c31..fd76252eb36 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -91,9 +91,9 @@ module Gitlab def relation_class case @importable - when Project + when ::Project ProjectMember - when Group + when ::Group GroupMember end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 4fa909ac94b..4fa909ac94b 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb new file mode 100644 index 00000000000..c3637b1c115 --- /dev/null +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + # Given a class, it finds or creates a new object + # (initializes in the case of Label) at group or project level. + # If it does not exist in the group, it creates it at project level. + # + # Example: + # `ObjectBuilder.build(Label, label_attributes)` + # finds or initializes a label with the given attributes. + # + # It also adds some logic around Group Labels/Milestones for edge cases. + class ObjectBuilder < Base::ObjectBuilder + def self.build(*args) + ::Project.transaction do + super + end + end + + def initialize(klass, attributes) + super + + @group = @attributes['group'] + @project = @attributes['project'] + end + + def find + return if epic? && group.nil? + + super + end + + private + + attr_reader :group, :project + + def where_clauses + [ + where_clause_base, + where_clause_for_title, + where_clause_for_klass + ].compact + end + + # Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present + # For example: merge_request has :target_project_id, and we are searching by :iid + # or, if group is present: + # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}` + def where_clause_base + [].tap do |clauses| + clauses << table[:project_id].eq(project.id) if project + clauses << table[:group_id].in(group.self_and_ancestors_ids) if group + end.reduce(:or) + end + + # Returns Arel clause for a particular model or `nil`. + def where_clause_for_klass + attrs_to_arel(attributes.slice('iid')) if merge_request? + end + + def prepare_attributes + attributes.dup.tap do |atts| + atts.delete('group') unless epic? + + if label? + atts['type'] = 'ProjectLabel' # Always create project labels + elsif milestone? + if atts['group_id'] # Transform new group milestones into project ones + atts['iid'] = nil + atts.delete('group_id') + else + claim_iid + end + end + + atts['importing'] = true if klass.ancestors.include?(Importable) + end + end + + def label? + klass == Label + end + + def milestone? + klass == Milestone + end + + def merge_request? + klass == MergeRequest + end + + def epic? + klass == Epic + end + + # If an existing group milestone used the IID + # claim the IID back and set the group milestone to use one available + # This is necessary to fix situations like the following: + # - Importing into a user namespace project with exported group milestones + # where the IID of the Group milestone could conflict with a project one. + def claim_iid + # The milestone has to be a group milestone, as it's the only case where + # we set the IID as the maximum. The rest of them are fixed. + milestone = project.milestones.find_by(iid: attributes['iid']) + + return unless milestone + + milestone.iid = nil + milestone.ensure_project_iid! + milestone.save! + end + end + end + end +end + +Gitlab::ImportExport::Project::ObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::Project::ObjectBuilder') diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb new file mode 100644 index 00000000000..951482a933a --- /dev/null +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class RelationFactory < Base::RelationFactory + prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule + + OVERRIDES = { snippets: :project_snippets, + ci_pipelines: 'Ci::Pipeline', + pipelines: 'Ci::Pipeline', + stages: 'Ci::Stage', + statuses: 'commit_status', + triggers: 'Ci::Trigger', + pipeline_schedules: 'Ci::PipelineSchedule', + builds: 'Ci::Build', + runners: 'Ci::Runner', + hooks: 'ProjectHook', + merge_access_levels: 'ProtectedBranch::MergeAccessLevel', + push_access_levels: 'ProtectedBranch::PushAccessLevel', + create_access_levels: 'ProtectedTag::CreateAccessLevel', + labels: :project_labels, + priorities: :label_priorities, + auto_devops: :project_auto_devops, + label: :project_label, + custom_attributes: 'ProjectCustomAttribute', + project_badges: 'Badge', + metrics: 'MergeRequest::Metrics', + ci_cd_settings: 'ProjectCiCdSetting', + error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', + links: 'Releases::Link', + metrics_setting: 'ProjectMetricsSetting' }.freeze + + BUILD_MODELS = %i[Ci::Build commit_status].freeze + + GROUP_REFERENCES = %w[group_id].freeze + + PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze + + EXISTING_OBJECT_RELATIONS = %i[ + milestone + milestones + label + labels + project_label + project_labels + group_label + group_labels + project_feature + merge_request + epic + ProjectCiCdSetting + container_expiration_policy + ].freeze + + def create + @object = super + + # We preload the project, user, and group to re-use objects + @object = preload_keys(@object, PROJECT_REFERENCES, @importable) + @object = preload_keys(@object, GROUP_REFERENCES, @importable.group) + @object = preload_keys(@object, USER_REFERENCES, @user) + end + + private + + def invalid_relation? + # Do not create relation if it is: + # - An unknown service + # - A legacy trigger + unknown_service? || + (!Feature.enabled?(:use_legacy_pipeline_triggers, @importable) && legacy_trigger?) + end + + def setup_models + case @relation_name + when :merge_request_diff_files then setup_diff + when :notes then setup_note + when :'Ci::Pipeline' then setup_pipeline + when *BUILD_MODELS then setup_build + end + + update_project_references + update_group_references + end + + def generate_imported_object + if @relation_name == :merge_requests + MergeRequestParser.new(@importable, @relation_hash.delete('diff_head_sha'), super, @relation_hash).parse! + else + super + end + end + + def update_project_references + # If source and target are the same, populate them with the new project ID. + if @relation_hash['source_project_id'] + @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID + end + + @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id'] + end + + def same_source_and_target? + @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] + end + + def update_group_references + return unless existing_object? + return unless @relation_hash['group_id'] + + @relation_hash['group_id'] = @importable.namespace_id + end + + def setup_build + @relation_hash.delete('trace') # old export files have trace + @relation_hash.delete('token') + @relation_hash.delete('commands') + @relation_hash.delete('artifacts_file_store') + @relation_hash.delete('artifacts_metadata_store') + @relation_hash.delete('artifacts_size') + end + + def setup_diff + @relation_hash['diff'] = @relation_hash.delete('utf8_diff') + end + + def setup_pipeline + @relation_hash.fetch('stages', []).each do |stage| + stage.statuses.each do |status| + status.pipeline = imported_object + end + end + end + + def unknown_service? + @relation_name == :services && parsed_relation_hash['type'] && + !Object.const_defined?(parsed_relation_hash['type']) + end + + def legacy_trigger? + @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil? + end + + def preload_keys(object, references, value) + return object unless value + + references.each do |key| + attribute = "#{key.delete_suffix('_id')}=".to_sym + next unless object.respond_to?(key) && object.respond_to?(attribute) + + if object.read_attribute(key) == value&.id + object.public_send(attribute, value) # rubocop:disable GitlabSecurity/PublicSend + end + end + + object + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_loader.rb b/lib/gitlab/import_export/project/tree_loader.rb new file mode 100644 index 00000000000..6d4737a2d00 --- /dev/null +++ b/lib/gitlab/import_export/project/tree_loader.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class TreeLoader + def load(path, dedup_entries: false) + tree_hash = ActiveSupport::JSON.decode(IO.read(path)) + + if dedup_entries + dedup_tree(tree_hash) + else + tree_hash + end + end + + private + + # This function removes duplicate entries from the given tree recursively + # by caching nodes it encounters repeatedly. We only consider nodes for + # which there can actually be multiple equivalent instances (e.g. strings, + # hashes and arrays, but not `nil`s, numbers or booleans.) + # + # The algorithm uses a recursive depth-first descent with 3 cases, starting + # with a root node (the tree/hash itself): + # - a node has already been cached; in this case we return it from the cache + # - a node has not been cached yet but should be; descend into its children + # - a node is neither cached nor qualifies for caching; this is a no-op + def dedup_tree(node, nodes_seen = {}) + if nodes_seen.key?(node) && distinguishable?(node) + yield nodes_seen[node] + elsif should_dedup?(node) + nodes_seen[node] = node + + case node + when Array + node.each_index do |idx| + dedup_tree(node[idx], nodes_seen) do |cached_node| + node[idx] = cached_node + end + end + when Hash + node.each do |k, v| + dedup_tree(v, nodes_seen) do |cached_node| + node[k] = cached_node + end + end + end + else + node + end + end + + # We do not need to consider nodes for which there cannot be multiple instances + def should_dedup?(node) + node && !(node.is_a?(Numeric) || node.is_a?(TrueClass) || node.is_a?(FalseClass)) + end + + # We can only safely de-dup values that are distinguishable. True value objects + # are always distinguishable by nature. Hashes however can represent entities, + # which are identified by ID, not value. We therefore disallow de-duping hashes + # that do not have an `id` field, since we might risk dropping entities that + # have equal attributes yet different identities. + def distinguishable?(node) + if node.is_a?(Hash) + node.key?('id') + else + true + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb new file mode 100644 index 00000000000..a5123f16dbc --- /dev/null +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class TreeRestorer + LARGE_PROJECT_FILE_SIZE_BYTES = 500.megabyte + + attr_reader :user + attr_reader :shared + attr_reader :project + + def initialize(user:, shared:, project:) + @user = user + @shared = shared + @project = project + @tree_loader = TreeLoader.new + end + + def restore + @tree_hash = read_tree_hash + @project_members = @tree_hash.delete('project_members') + + RelationRenameService.rename(@tree_hash) + + if relation_tree_restorer.restore + import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do + @project.merge_requests.set_latest_merge_request_diff_ids! + end + + true + else + false + end + rescue => e + @shared.error(e) + false + end + + private + + def large_project?(path) + File.size(path) >= LARGE_PROJECT_FILE_SIZE_BYTES + end + + def read_tree_hash + path = File.join(@shared.export_path, 'project.json') + dedup_entries = large_project?(path) && + Feature.enabled?(:dedup_project_import_metadata, project.group) + + @tree_loader.load(path, dedup_entries: dedup_entries) + rescue => e + Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + + def relation_tree_restorer + @relation_tree_restorer ||= RelationTreeRestorer.new( + user: @user, + shared: @shared, + importable: @project, + tree_hash: @tree_hash, + object_builder: object_builder, + members_mapper: members_mapper, + relation_factory: relation_factory, + reader: reader + ) + end + + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, + user: @user, + importable: @project) + end + + def object_builder + Project::ObjectBuilder + end + + def relation_factory + Project::RelationFactory + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end + + def import_failure_service + @import_failure_service ||= ImportFailureService.new(@project) + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb new file mode 100644 index 00000000000..58f33a04851 --- /dev/null +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class TreeSaver + attr_reader :full_path + + def initialize(project:, current_user:, shared:, params: {}) + @params = params + @project = project + @current_user = current_user + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) + end + + def save + project_tree = tree_saver.serialize(@project, reader.project_tree) + fix_project_tree(project_tree) + tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) + + true + rescue => e + @shared.error(e) + false + end + + private + + # Aware that the resulting hash needs to be pure-hash and + # does not include any AR objects anymore, only objects that run `.to_json` + def fix_project_tree(project_tree) + if @params[:description].present? + project_tree['description'] = @params[:description] + end + + project_tree['project_members'] += group_members_array + + RelationRenameService.add_new_associations(project_tree) + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end + + def group_members_array + group_members.as_json(reader.group_members_tree).each do |group_member| + group_member['source_type'] = 'Project' # Make group members project members of the future import + end + end + + def group_members + return [] unless @current_user.can?(:admin_group, @project.group) + + # We need `.where.not(user_id: nil)` here otherwise when a group has an + # invitee, it would make the following query return 0 rows since a NULL + # user_id would be present in the subquery + # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values + non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id) + + GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) + end + + def tree_saver + @tree_saver ||= RelationTreeSaver.new + end + end + end + end +end diff --git a/lib/gitlab/import_export/project_relation_factory.rb b/lib/gitlab/import_export/project_relation_factory.rb deleted file mode 100644 index 0e08a66b89c..00000000000 --- a/lib/gitlab/import_export/project_relation_factory.rb +++ /dev/null @@ -1,160 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class ProjectRelationFactory < BaseRelationFactory - prepend_if_ee('::EE::Gitlab::ImportExport::ProjectRelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule - - OVERRIDES = { snippets: :project_snippets, - ci_pipelines: 'Ci::Pipeline', - pipelines: 'Ci::Pipeline', - stages: 'Ci::Stage', - statuses: 'commit_status', - triggers: 'Ci::Trigger', - pipeline_schedules: 'Ci::PipelineSchedule', - builds: 'Ci::Build', - runners: 'Ci::Runner', - hooks: 'ProjectHook', - merge_access_levels: 'ProtectedBranch::MergeAccessLevel', - push_access_levels: 'ProtectedBranch::PushAccessLevel', - create_access_levels: 'ProtectedTag::CreateAccessLevel', - labels: :project_labels, - priorities: :label_priorities, - auto_devops: :project_auto_devops, - label: :project_label, - custom_attributes: 'ProjectCustomAttribute', - project_badges: 'Badge', - metrics: 'MergeRequest::Metrics', - ci_cd_settings: 'ProjectCiCdSetting', - error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', - links: 'Releases::Link', - metrics_setting: 'ProjectMetricsSetting' }.freeze - - BUILD_MODELS = %i[Ci::Build commit_status].freeze - - GROUP_REFERENCES = %w[group_id].freeze - - PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze - - EXISTING_OBJECT_RELATIONS = %i[ - milestone - milestones - label - labels - project_label - project_labels - group_label - group_labels - project_feature - merge_request - epic - ProjectCiCdSetting - container_expiration_policy - ].freeze - - def create - @object = super - - # We preload the project, user, and group to re-use objects - @object = preload_keys(@object, PROJECT_REFERENCES, @importable) - @object = preload_keys(@object, GROUP_REFERENCES, @importable.group) - @object = preload_keys(@object, USER_REFERENCES, @user) - end - - private - - def invalid_relation? - # Do not create relation if it is: - # - An unknown service - # - A legacy trigger - unknown_service? || - (!Feature.enabled?(:use_legacy_pipeline_triggers, @importable) && legacy_trigger?) - end - - def setup_models - case @relation_name - when :merge_request_diff_files then setup_diff - when :notes then setup_note - when :'Ci::Pipeline' then setup_pipeline - when *BUILD_MODELS then setup_build - end - - update_project_references - update_group_references - end - - def generate_imported_object - if @relation_name == :merge_requests - MergeRequestParser.new(@importable, @relation_hash.delete('diff_head_sha'), super, @relation_hash).parse! - else - super - end - end - - def update_project_references - # If source and target are the same, populate them with the new project ID. - if @relation_hash['source_project_id'] - @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID - end - - @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id'] - end - - def same_source_and_target? - @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] - end - - def update_group_references - return unless existing_object? - return unless @relation_hash['group_id'] - - @relation_hash['group_id'] = @importable.namespace_id - end - - def setup_build - @relation_hash.delete('trace') # old export files have trace - @relation_hash.delete('token') - @relation_hash.delete('commands') - @relation_hash.delete('artifacts_file_store') - @relation_hash.delete('artifacts_metadata_store') - @relation_hash.delete('artifacts_size') - end - - def setup_diff - @relation_hash['diff'] = @relation_hash.delete('utf8_diff') - end - - def setup_pipeline - @relation_hash.fetch('stages', []).each do |stage| - stage.statuses.each do |status| - status.pipeline = imported_object - end - end - end - - def unknown_service? - @relation_name == :services && parsed_relation_hash['type'] && - !Object.const_defined?(parsed_relation_hash['type']) - end - - def legacy_trigger? - @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil? - end - - def preload_keys(object, references, value) - return object unless value - - references.each do |key| - attribute = "#{key.delete_suffix('_id')}=".to_sym - next unless object.respond_to?(key) && object.respond_to?(attribute) - - if object.read_attribute(key) == value&.id - object.public_send(attribute, value) # rubocop:disable GitlabSecurity/PublicSend - end - end - - object - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_loader.rb b/lib/gitlab/import_export/project_tree_loader.rb deleted file mode 100644 index fc21858043d..00000000000 --- a/lib/gitlab/import_export/project_tree_loader.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class ProjectTreeLoader - def load(path, dedup_entries: false) - tree_hash = ActiveSupport::JSON.decode(IO.read(path)) - - if dedup_entries - dedup_tree(tree_hash) - else - tree_hash - end - end - - private - - # This function removes duplicate entries from the given tree recursively - # by caching nodes it encounters repeatedly. We only consider nodes for - # which there can actually be multiple equivalent instances (e.g. strings, - # hashes and arrays, but not `nil`s, numbers or booleans.) - # - # The algorithm uses a recursive depth-first descent with 3 cases, starting - # with a root node (the tree/hash itself): - # - a node has already been cached; in this case we return it from the cache - # - a node has not been cached yet but should be; descend into its children - # - a node is neither cached nor qualifies for caching; this is a no-op - def dedup_tree(node, nodes_seen = {}) - if nodes_seen.key?(node) && distinguishable?(node) - yield nodes_seen[node] - elsif should_dedup?(node) - nodes_seen[node] = node - - case node - when Array - node.each_index do |idx| - dedup_tree(node[idx], nodes_seen) do |cached_node| - node[idx] = cached_node - end - end - when Hash - node.each do |k, v| - dedup_tree(v, nodes_seen) do |cached_node| - node[k] = cached_node - end - end - end - else - node - end - end - - # We do not need to consider nodes for which there cannot be multiple instances - def should_dedup?(node) - node && !(node.is_a?(Numeric) || node.is_a?(TrueClass) || node.is_a?(FalseClass)) - end - - # We can only safely de-dup values that are distinguishable. True value objects - # are always distinguishable by nature. Hashes however can represent entities, - # which are identified by ID, not value. We therefore disallow de-duping hashes - # that do not have an `id` field, since we might risk dropping entities that - # have equal attributes yet different identities. - def distinguishable?(node) - if node.is_a?(Hash) - node.key?('id') - else - true - end - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb deleted file mode 100644 index aae07657ea0..00000000000 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ /dev/null @@ -1,92 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class ProjectTreeRestorer - LARGE_PROJECT_FILE_SIZE_BYTES = 500.megabyte - - attr_reader :user - attr_reader :shared - attr_reader :project - - def initialize(user:, shared:, project:) - @user = user - @shared = shared - @project = project - @tree_loader = ProjectTreeLoader.new - end - - def restore - @tree_hash = read_tree_hash - @project_members = @tree_hash.delete('project_members') - - RelationRenameService.rename(@tree_hash) - - if relation_tree_restorer.restore - import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do - @project.merge_requests.set_latest_merge_request_diff_ids! - end - - true - else - false - end - rescue => e - @shared.error(e) - false - end - - private - - def large_project?(path) - File.size(path) >= LARGE_PROJECT_FILE_SIZE_BYTES - end - - def read_tree_hash - path = File.join(@shared.export_path, 'project.json') - dedup_entries = large_project?(path) && - Feature.enabled?(:dedup_project_import_metadata, project.group) - - @tree_loader.load(path, dedup_entries: dedup_entries) - rescue => e - Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') - end - - def relation_tree_restorer - @relation_tree_restorer ||= RelationTreeRestorer.new( - user: @user, - shared: @shared, - importable: @project, - tree_hash: @tree_hash, - object_builder: object_builder, - members_mapper: members_mapper, - relation_factory: relation_factory, - reader: reader - ) - end - - def members_mapper - @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, - user: @user, - importable: @project) - end - - def object_builder - Gitlab::ImportExport::GroupProjectObjectBuilder - end - - def relation_factory - Gitlab::ImportExport::ProjectRelationFactory - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) - end - - def import_failure_service - @import_failure_service ||= ImportFailureService.new(@project) - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb deleted file mode 100644 index 386a4cfdfc6..00000000000 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ /dev/null @@ -1,68 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class ProjectTreeSaver - attr_reader :full_path - - def initialize(project:, current_user:, shared:, params: {}) - @params = params - @project = project - @current_user = current_user - @shared = shared - @full_path = File.join(@shared.export_path, ImportExport.project_filename) - end - - def save - project_tree = tree_saver.serialize(@project, reader.project_tree) - fix_project_tree(project_tree) - tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) - - true - rescue => e - @shared.error(e) - false - end - - private - - # Aware that the resulting hash needs to be pure-hash and - # does not include any AR objects anymore, only objects that run `.to_json` - def fix_project_tree(project_tree) - if @params[:description].present? - project_tree['description'] = @params[:description] - end - - project_tree['project_members'] += group_members_array - - RelationRenameService.add_new_associations(project_tree) - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) - end - - def group_members_array - group_members.as_json(reader.group_members_tree).each do |group_member| - group_member['source_type'] = 'Project' # Make group members project members of the future import - end - end - - def group_members - return [] unless @current_user.can?(:admin_group, @project.group) - - # We need `.where.not(user_id: nil)` here otherwise when a group has an - # invitee, it would make the following query return 0 rows since a NULL - # user_id would be present in the subquery - # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values - non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id) - - GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) - end - - def tree_saver - @tree_saver ||= RelationTreeSaver.new - end - end - end -end diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 9b84ade1525..1797bbad51a 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -69,7 +69,7 @@ module Gitlab def process_relation_item!(relation_key, relation_definition, relation_index, data_hash) relation_object = build_relation(relation_key, relation_definition, data_hash) return unless relation_object - return if importable_class == Project && group_model?(relation_object) + return if importable_class == ::Project && group_model?(relation_object) relation_object.assign_attributes(importable_class_sym => @importable) @@ -110,7 +110,7 @@ module Gitlab excluded_keys: excluded_keys_for_relation(importable_class_sym)) @importable.assign_attributes(params) - @importable.drop_visibility_level! if importable_class == Project + @importable.drop_visibility_level! if importable_class == ::Project Gitlab::Timeless.timeless(@importable) do @importable.save! diff --git a/lib/gitlab_danger.rb b/lib/gitlab_danger.rb index e776e2b7ea3..cf57b20790b 100644 --- a/lib/gitlab_danger.rb +++ b/lib/gitlab_danger.rb @@ -3,7 +3,6 @@ class GitlabDanger LOCAL_RULES ||= %w[ changes_size - gemfile documentation frozen_string duplicate_yarn_dependencies |