From f20912df033d07c46b0989012244d96d0a12b66d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Fri, 6 Apr 2018 15:23:49 +0000 Subject: Extend API for importing a project export with overwrite support --- app/models/group.rb | 4 ++ app/models/namespace.rb | 4 ++ app/models/project.rb | 4 +- .../projects/base_move_relations_service.rb | 22 +++++++ app/services/projects/destroy_service.rb | 26 ++++++-- .../projects/gitlab_projects_import_service.rb | 23 +++++++- app/services/projects/move_access_service.rb | 25 ++++++++ .../projects/move_deploy_keys_projects_service.rb | 31 ++++++++++ app/services/projects/move_forks_service.rb | 42 +++++++++++++ .../projects/move_lfs_objects_projects_service.rb | 29 +++++++++ .../projects/move_notification_settings_service.rb | 38 ++++++++++++ .../move_project_authorizations_service.rb | 40 +++++++++++++ .../projects/move_project_group_links_service.rb | 40 +++++++++++++ .../projects/move_project_members_service.rb | 40 +++++++++++++ .../projects/move_users_star_projects_service.rb | 20 +++++++ app/services/projects/overwrite_project_service.rb | 69 ++++++++++++++++++++++ 16 files changed, 451 insertions(+), 6 deletions(-) create mode 100644 app/services/projects/base_move_relations_service.rb create mode 100644 app/services/projects/move_access_service.rb create mode 100644 app/services/projects/move_deploy_keys_projects_service.rb create mode 100644 app/services/projects/move_forks_service.rb create mode 100644 app/services/projects/move_lfs_objects_projects_service.rb create mode 100644 app/services/projects/move_notification_settings_service.rb create mode 100644 app/services/projects/move_project_authorizations_service.rb create mode 100644 app/services/projects/move_project_group_links_service.rb create mode 100644 app/services/projects/move_project_members_service.rb create mode 100644 app/services/projects/move_users_star_projects_service.rb create mode 100644 app/services/projects/overwrite_project_service.rb (limited to 'app') diff --git a/app/models/group.rb b/app/models/group.rb index 3cfe21ac93b..8ff781059cc 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -286,6 +286,10 @@ class Group < Namespace false end + def refresh_project_authorizations + refresh_members_authorized_projects(blocking: false) + end + private def update_two_factor_requirement diff --git a/app/models/namespace.rb b/app/models/namespace.rb index e350b675639..2b63aa33222 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -252,6 +252,10 @@ class Namespace < ActiveRecord::Base [] end + def refresh_project_authorizations + owner.refresh_authorized_projects + end + private def path_or_parent_changed? diff --git a/app/models/project.rb b/app/models/project.rb index 1b29cbf28d2..96907f3b23d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1472,7 +1472,9 @@ class Project < ActiveRecord::Base end def rename_repo_notify! - send_move_instructions(full_path_was) + # When we import a project overwriting the original project, there + # is a move operation. In that case we don't want to send the instructions. + send_move_instructions(full_path_was) unless started? expires_full_path_cache self.old_path_with_namespace = full_path_was diff --git a/app/services/projects/base_move_relations_service.rb b/app/services/projects/base_move_relations_service.rb new file mode 100644 index 00000000000..e8fd3ef57e5 --- /dev/null +++ b/app/services/projects/base_move_relations_service.rb @@ -0,0 +1,22 @@ +module Projects + class BaseMoveRelationsService < BaseService + attr_reader :source_project + def execute(source_project, remove_remaining_elements: true) + return if source_project.blank? + + @source_project = source_project + + true + end + + private + + def prepare_relation(relation, id_param = :id) + if Gitlab::Database.postgresql? + relation + else + relation.model.where("#{id_param}": relation.pluck(id_param)) + end + end + end +end diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index 114762c208e..aa14206db3b 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -46,6 +46,20 @@ module Projects raise end + def attempt_repositories_rollback + return unless @project + + flush_caches(@project) + + unless mv_repository(removal_path(repo_path), repo_path) + raise_error('Failed to restore project repository. Please contact the administrator.') + end + + unless mv_repository(removal_path(wiki_path), wiki_path) + raise_error('Failed to restore wiki repository. Please contact the administrator.') + end + end + private def repo_path @@ -70,12 +84,9 @@ module Projects # Skip repository removal. We use this flag when remove user or group return true if params[:skip_repo] == true - # There is a possibility project does not have repository or wiki - return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git') - new_path = removal_path(path) - if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path) + if mv_repository(path, new_path) log_info("Repository \"#{path}\" moved to \"#{new_path}\"") project.run_after_commit do @@ -87,6 +98,13 @@ module Projects end end + def mv_repository(from_path, to_path) + # There is a possibility project does not have repository or wiki + return true unless gitlab_shell.exists?(project.repository_storage_path, from_path + '.git') + + gitlab_shell.mv_repository(project.repository_storage_path, from_path, to_path) + end + def attempt_rollback(project, message) return unless project diff --git a/app/services/projects/gitlab_projects_import_service.rb b/app/services/projects/gitlab_projects_import_service.rb index fb4afb85588..a16268f4fd2 100644 --- a/app/services/projects/gitlab_projects_import_service.rb +++ b/app/services/projects/gitlab_projects_import_service.rb @@ -15,9 +15,18 @@ module Projects file = params.delete(:file) FileUtils.copy_entry(file.path, import_upload_path) + @overwrite = params.delete(:overwrite) + data = {} + data[:override_params] = @override_params if @override_params + + if overwrite_project? + data[:original_path] = params[:path] + params[:path] += "-#{tmp_filename}" + end + params[:import_type] = 'gitlab_project' params[:import_source] = import_upload_path - params[:import_data] = { data: { override_params: @override_params } } if @override_params + params[:import_data] = { data: data } if data.present? ::Projects::CreateService.new(current_user, params).execute end @@ -31,5 +40,17 @@ module Projects def tmp_filename SecureRandom.hex end + + def overwrite_project? + @overwrite && project_with_same_full_path? + end + + def project_with_same_full_path? + Project.find_by_full_path("#{current_namespace.full_path}/#{params[:path]}").present? + end + + def current_namespace + @current_namespace ||= Namespace.find_by(id: params[:namespace_id]) + end end end diff --git a/app/services/projects/move_access_service.rb b/app/services/projects/move_access_service.rb new file mode 100644 index 00000000000..3af3a22d486 --- /dev/null +++ b/app/services/projects/move_access_service.rb @@ -0,0 +1,25 @@ +module Projects + class MoveAccessService < BaseMoveRelationsService + def execute(source_project, remove_remaining_elements: true) + return unless super + + @project.with_transaction_returning_status do + if @project.namespace != source_project.namespace + @project.run_after_commit do + source_project.namespace.refresh_project_authorizations + self.namespace.refresh_project_authorizations + end + end + + ::Projects::MoveProjectMembersService.new(@project, @current_user) + .execute(source_project, remove_remaining_elements: remove_remaining_elements) + ::Projects::MoveProjectGroupLinksService.new(@project, @current_user) + .execute(source_project, remove_remaining_elements: remove_remaining_elements) + ::Projects::MoveProjectAuthorizationsService.new(@project, @current_user) + .execute(source_project, remove_remaining_elements: remove_remaining_elements) + + success + end + end + end +end diff --git a/app/services/projects/move_deploy_keys_projects_service.rb b/app/services/projects/move_deploy_keys_projects_service.rb new file mode 100644 index 00000000000..dde420655b0 --- /dev/null +++ b/app/services/projects/move_deploy_keys_projects_service.rb @@ -0,0 +1,31 @@ +module Projects + class MoveDeployKeysProjectsService < BaseMoveRelationsService + def execute(source_project, remove_remaining_elements: true) + return unless super + + Project.transaction(requires_new: true) do + move_deploy_keys_projects + remove_remaining_deploy_keys_projects if remove_remaining_elements + + success + end + end + + private + + def move_deploy_keys_projects + prepare_relation(non_existent_deploy_keys_projects) + .update_all(project_id: @project.id) + end + + def non_existent_deploy_keys_projects + source_project.deploy_keys_projects + .joins(:deploy_key) + .where.not(keys: { fingerprint: @project.deploy_keys.select(:fingerprint) }) + end + + def remove_remaining_deploy_keys_projects + source_project.deploy_keys_projects.destroy_all + end + end +end diff --git a/app/services/projects/move_forks_service.rb b/app/services/projects/move_forks_service.rb new file mode 100644 index 00000000000..d2901ea1457 --- /dev/null +++ b/app/services/projects/move_forks_service.rb @@ -0,0 +1,42 @@ +module Projects + class MoveForksService < BaseMoveRelationsService + def execute(source_project, remove_remaining_elements: true) + return unless super && source_project.fork_network + + Project.transaction(requires_new: true) do + move_forked_project_links + move_fork_network_members + update_root_project + refresh_forks_count + + success + end + end + + private + + def move_forked_project_links + # Update ancestor + ForkedProjectLink.where(forked_to_project: source_project) + .update_all(forked_to_project_id: @project.id) + + # Update the descendants + ForkedProjectLink.where(forked_from_project: source_project) + .update_all(forked_from_project_id: @project.id) + end + + def move_fork_network_members + ForkNetworkMember.where(project: source_project).update_all(project_id: @project.id) + ForkNetworkMember.where(forked_from_project: source_project).update_all(forked_from_project_id: @project.id) + end + + def update_root_project + # Update root network project + ForkNetwork.where(root_project: source_project).update_all(root_project_id: @project.id) + end + + def refresh_forks_count + Projects::ForksCountService.new(@project).refresh_cache + end + end +end diff --git a/app/services/projects/move_lfs_objects_projects_service.rb b/app/services/projects/move_lfs_objects_projects_service.rb new file mode 100644 index 00000000000..298da5f1a82 --- /dev/null +++ b/app/services/projects/move_lfs_objects_projects_service.rb @@ -0,0 +1,29 @@ +module Projects + class MoveLfsObjectsProjectsService < BaseMoveRelationsService + def execute(source_project, remove_remaining_elements: true) + return unless super + + Project.transaction(requires_new: true) do + move_lfs_objects_projects + remove_remaining_lfs_objects_project if remove_remaining_elements + + success + end + end + + private + + def move_lfs_objects_projects + prepare_relation(non_existent_lfs_objects_projects) + .update_all(project_id: @project.lfs_storage_project.id) + end + + def remove_remaining_lfs_objects_project + source_project.lfs_objects_projects.destroy_all + end + + def non_existent_lfs_objects_projects + source_project.lfs_objects_projects.where.not(lfs_object: @project.lfs_objects) + end + end +end diff --git a/app/services/projects/move_notification_settings_service.rb b/app/services/projects/move_notification_settings_service.rb new file mode 100644 index 00000000000..f7be461a5da --- /dev/null +++ b/app/services/projects/move_notification_settings_service.rb @@ -0,0 +1,38 @@ +module Projects + class MoveNotificationSettingsService < BaseMoveRelationsService + def execute(source_project, remove_remaining_elements: true) + return unless super + + Project.transaction(requires_new: true) do + move_notification_settings + remove_remaining_notification_settings if remove_remaining_elements + + success + end + end + + private + + def move_notification_settings + prepare_relation(non_existent_notifications) + .update_all(source_id: @project.id) + end + + # Remove remaining notification settings from source_project + def remove_remaining_notification_settings + source_project.notification_settings.destroy_all + end + + # Get users of current notification_settings + def users_in_target_project + @project.notification_settings.select(:user_id) + end + + # Look for notification_settings in source_project that are not in the target project + def non_existent_notifications + source_project.notification_settings + .select(:id) + .where.not(user_id: users_in_target_project) + end + end +end diff --git a/app/services/projects/move_project_authorizations_service.rb b/app/services/projects/move_project_authorizations_service.rb new file mode 100644 index 00000000000..5ef12fc49e5 --- /dev/null +++ b/app/services/projects/move_project_authorizations_service.rb @@ -0,0 +1,40 @@ +# NOTE: This service cannot be used directly because it is part of a +# a bigger process. Instead, use the service MoveAccessService which moves +# project memberships, project group links, authorizations and refreshes +# the authorizations if neccessary +module Projects + class MoveProjectAuthorizationsService < BaseMoveRelationsService + def execute(source_project, remove_remaining_elements: true) + return unless super + + Project.transaction(requires_new: true) do + move_project_authorizations + + remove_remaining_authorizations if remove_remaining_elements + + success + end + end + + private + + def move_project_authorizations + prepare_relation(non_existent_authorization, :user_id) + .update_all(project_id: @project.id) + end + + def remove_remaining_authorizations + # I think because the Project Authorization table does not have a primary key + # it brings a lot a problems/bugs. First, Rails raises PG::SyntaxException if we use + # destroy_all instead of delete_all. + source_project.project_authorizations.delete_all(:delete_all) + end + + # Look for authorizations in source_project that are not in the target project + def non_existent_authorization + source_project.project_authorizations + .select(:user_id) + .where.not(user: @project.authorized_users) + end + end +end diff --git a/app/services/projects/move_project_group_links_service.rb b/app/services/projects/move_project_group_links_service.rb new file mode 100644 index 00000000000..dbeffd7dae9 --- /dev/null +++ b/app/services/projects/move_project_group_links_service.rb @@ -0,0 +1,40 @@ +# NOTE: This service cannot be used directly because it is part of a +# a bigger process. Instead, use the service MoveAccessService which moves +# project memberships, project group links, authorizations and refreshes +# the authorizations if neccessary +module Projects + class MoveProjectGroupLinksService < BaseMoveRelationsService + def execute(source_project, remove_remaining_elements: true) + return unless super + + Project.transaction(requires_new: true) do + move_group_links + remove_remaining_project_group_links if remove_remaining_elements + + success + end + end + + private + + def move_group_links + prepare_relation(non_existent_group_links) + .update_all(project_id: @project.id) + end + + # Remove remaining project group links from source_project + def remove_remaining_project_group_links + source_project.reload.project_group_links.destroy_all + end + + def group_links_in_target_project + @project.project_group_links.select(:group_id) + end + + # Look for groups in source_project that are not in the target project + def non_existent_group_links + source_project.project_group_links + .where.not(group_id: group_links_in_target_project) + end + end +end diff --git a/app/services/projects/move_project_members_service.rb b/app/services/projects/move_project_members_service.rb new file mode 100644 index 00000000000..22a5f0a3fe6 --- /dev/null +++ b/app/services/projects/move_project_members_service.rb @@ -0,0 +1,40 @@ +# NOTE: This service cannot be used directly because it is part of a +# a bigger process. Instead, use the service MoveAccessService which moves +# project memberships, project group links, authorizations and refreshes +# the authorizations if neccessary +module Projects + class MoveProjectMembersService < BaseMoveRelationsService + def execute(source_project, remove_remaining_elements: true) + return unless super + + Project.transaction(requires_new: true) do + move_project_members + remove_remaining_members if remove_remaining_elements + + success + end + end + + private + + def move_project_members + prepare_relation(non_existent_members).update_all(source_id: @project.id) + end + + def remove_remaining_members + # Remove remaining members and authorizations from source_project + source_project.project_members.destroy_all + end + + def project_members_in_target_project + @project.project_members.select(:user_id) + end + + # Look for members in source_project that are not in the target project + def non_existent_members + source_project.members + .select(:id) + .where.not(user_id: @project.project_members.select(:user_id)) + end + end +end diff --git a/app/services/projects/move_users_star_projects_service.rb b/app/services/projects/move_users_star_projects_service.rb new file mode 100644 index 00000000000..079fd5b9685 --- /dev/null +++ b/app/services/projects/move_users_star_projects_service.rb @@ -0,0 +1,20 @@ +module Projects + class MoveUsersStarProjectsService < BaseMoveRelationsService + def execute(source_project, remove_remaining_elements: true) + return unless super + + user_stars = source_project.users_star_projects + + return unless user_stars.any? + + Project.transaction(requires_new: true) do + user_stars.update_all(project_id: @project.id) + + Project.reset_counters @project.id, :users_star_projects + Project.reset_counters source_project.id, :users_star_projects + + success + end + end + end +end diff --git a/app/services/projects/overwrite_project_service.rb b/app/services/projects/overwrite_project_service.rb new file mode 100644 index 00000000000..ce94f147aa9 --- /dev/null +++ b/app/services/projects/overwrite_project_service.rb @@ -0,0 +1,69 @@ +module Projects + class OverwriteProjectService < BaseService + def execute(source_project) + return unless source_project && source_project.namespace == @project.namespace + + Project.transaction do + move_before_destroy_relationships(source_project) + destroy_old_project(source_project) + rename_project(source_project.name, source_project.path) + + @project + end + # Projects::DestroyService can raise Exceptions, but we don't want + # to pass that kind of exception to the caller. Instead, we change it + # for a StandardError exception + rescue Exception => e # rubocop:disable Lint/RescueException + attempt_restore_repositories(source_project) + + if e.class == Exception + raise StandardError, e.message + else + raise + end + end + + private + + def move_before_destroy_relationships(source_project) + options = { remove_remaining_elements: false } + + ::Projects::MoveUsersStarProjectsService.new(@project, @current_user).execute(source_project, options) + ::Projects::MoveAccessService.new(@project, @current_user).execute(source_project, options) + ::Projects::MoveDeployKeysProjectsService.new(@project, @current_user).execute(source_project, options) + ::Projects::MoveNotificationSettingsService.new(@project, @current_user).execute(source_project, options) + ::Projects::MoveForksService.new(@project, @current_user).execute(source_project, options) + ::Projects::MoveLfsObjectsProjectsService.new(@project, @current_user).execute(source_project, options) + add_source_project_to_fork_network(source_project) + end + + def destroy_old_project(source_project) + # Delete previous project (synchronously) and unlink relations + ::Projects::DestroyService.new(source_project, @current_user).execute + end + + def rename_project(name, path) + # Update de project's name and path to the original name/path + ::Projects::UpdateService.new(@project, + @current_user, + { name: name, path: path }) + .execute + end + + def attempt_restore_repositories(project) + ::Projects::DestroyService.new(project, @current_user).attempt_repositories_rollback + end + + def add_source_project_to_fork_network(source_project) + return unless @project.fork_network + + # Because he have moved all references in the fork network from the source_project + # we won't be able to query the database (only through its cached data), + # for its former relationships. That's why we're adding it to the network + # as a fork of the target project + ForkNetworkMember.create!(fork_network: @project.fork_network, + project: source_project, + forked_from_project: @project) + end + end +end -- cgit v1.2.1