Extend API for importing a project export with overwrite support

16 files changed, 451 insertions, 6 deletions

diff --git a/app/models/group.rb b/app/models/group.rb
index 3cfe21ac93b..8ff781059cc 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -286,6 +286,10 @@ class Group < Namespace
     false
   end
 
+  def refresh_project_authorizations
+    refresh_members_authorized_projects(blocking: false)
+  end
+
   private
 
   def update_two_factor_requirement
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index e350b675639..2b63aa33222 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -252,6 +252,10 @@ class Namespace < ActiveRecord::Base
     []
   end
 
+  def refresh_project_authorizations
+    owner.refresh_authorized_projects
+  end
+
   private
 
   def path_or_parent_changed?
diff --git a/app/models/project.rb b/app/models/project.rb
index 1b29cbf28d2..96907f3b23d 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -1472,7 +1472,9 @@ class Project < ActiveRecord::Base
   end
 
   def rename_repo_notify!
-    send_move_instructions(full_path_was)
+    # When we import a project overwriting the original project, there
+    # is a move operation. In that case we don't want to send the instructions.
+    send_move_instructions(full_path_was) unless started?
     expires_full_path_cache
 
     self.old_path_with_namespace = full_path_was
diff --git a/app/services/projects/base_move_relations_service.rb b/app/services/projects/base_move_relations_service.rb
new file mode 100644
index 00000000000..e8fd3ef57e5
--- /dev/null
+++ b/app/services/projects/base_move_relations_service.rb
@@ -0,0 +1,22 @@
+module Projects
+  class BaseMoveRelationsService < BaseService
+    attr_reader :source_project
+    def execute(source_project, remove_remaining_elements: true)
+      return if source_project.blank?
+
+      @source_project = source_project
+
+      true
+    end
+
+    private
+
+    def prepare_relation(relation, id_param = :id)
+      if Gitlab::Database.postgresql?
+        relation
+      else
+        relation.model.where("#{id_param}": relation.pluck(id_param))
+      end
+    end
+  end
+end
diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb
index 114762c208e..aa14206db3b 100644
--- a/app/services/projects/destroy_service.rb
+++ b/app/services/projects/destroy_service.rb
@@ -46,6 +46,20 @@ module Projects
       raise
     end
 
+    def attempt_repositories_rollback
+      return unless @project
+
+      flush_caches(@project)
+
+      unless mv_repository(removal_path(repo_path), repo_path)
+        raise_error('Failed to restore project repository. Please contact the administrator.')
+      end
+
+      unless mv_repository(removal_path(wiki_path), wiki_path)
+        raise_error('Failed to restore wiki repository. Please contact the administrator.')
+      end
+    end
+
     private
 
     def repo_path
@@ -70,12 +84,9 @@ module Projects
       # Skip repository removal. We use this flag when remove user or group
       return true if params[:skip_repo] == true
 
-      # There is a possibility project does not have repository or wiki
-      return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git')
-
       new_path = removal_path(path)
 
-      if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path)
+      if mv_repository(path, new_path)
         log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
 
         project.run_after_commit do
@@ -87,6 +98,13 @@ module Projects
       end
     end
 
+    def mv_repository(from_path, to_path)
+      # There is a possibility project does not have repository or wiki
+      return true unless gitlab_shell.exists?(project.repository_storage_path, from_path + '.git')
+
+      gitlab_shell.mv_repository(project.repository_storage_path, from_path, to_path)
+    end
+
     def attempt_rollback(project, message)
       return unless project
 
diff --git a/app/services/projects/gitlab_projects_import_service.rb b/app/services/projects/gitlab_projects_import_service.rb
index fb4afb85588..a16268f4fd2 100644
--- a/app/services/projects/gitlab_projects_import_service.rb
+++ b/app/services/projects/gitlab_projects_import_service.rb
@@ -15,9 +15,18 @@ module Projects
       file = params.delete(:file)
       FileUtils.copy_entry(file.path, import_upload_path)
 
+      @overwrite = params.delete(:overwrite)
+      data = {}
+      data[:override_params] = @override_params if @override_params
+
+      if overwrite_project?
+        data[:original_path] = params[:path]
+        params[:path] += "-#{tmp_filename}"
+      end
+
       params[:import_type] = 'gitlab_project'
       params[:import_source] = import_upload_path
-      params[:import_data] = { data: { override_params: @override_params } } if @override_params
+      params[:import_data] = { data: data } if data.present?
 
       ::Projects::CreateService.new(current_user, params).execute
     end
@@ -31,5 +40,17 @@ module Projects
     def tmp_filename
       SecureRandom.hex
     end
+
+    def overwrite_project?
+      @overwrite && project_with_same_full_path?
+    end
+
+    def project_with_same_full_path?
+      Project.find_by_full_path("#{current_namespace.full_path}/#{params[:path]}").present?
+    end
+
+    def current_namespace
+      @current_namespace ||= Namespace.find_by(id: params[:namespace_id])
+    end
   end
 end
diff --git a/app/services/projects/move_access_service.rb b/app/services/projects/move_access_service.rb
new file mode 100644
index 00000000000..3af3a22d486
--- /dev/null
+++ b/app/services/projects/move_access_service.rb
@@ -0,0 +1,25 @@
+module Projects
+  class MoveAccessService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      @project.with_transaction_returning_status do
+        if @project.namespace != source_project.namespace
+          @project.run_after_commit do
+            source_project.namespace.refresh_project_authorizations
+            self.namespace.refresh_project_authorizations
+          end
+        end
+
+        ::Projects::MoveProjectMembersService.new(@project, @current_user)
+          .execute(source_project, remove_remaining_elements: remove_remaining_elements)
+        ::Projects::MoveProjectGroupLinksService.new(@project, @current_user)
+          .execute(source_project, remove_remaining_elements: remove_remaining_elements)
+        ::Projects::MoveProjectAuthorizationsService.new(@project, @current_user)
+          .execute(source_project, remove_remaining_elements: remove_remaining_elements)
+
+        success
+      end
+    end
+  end
+end
diff --git a/app/services/projects/move_deploy_keys_projects_service.rb b/app/services/projects/move_deploy_keys_projects_service.rb
new file mode 100644
index 00000000000..dde420655b0
--- /dev/null
+++ b/app/services/projects/move_deploy_keys_projects_service.rb
@@ -0,0 +1,31 @@
+module Projects
+  class MoveDeployKeysProjectsService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      Project.transaction(requires_new: true) do
+        move_deploy_keys_projects
+        remove_remaining_deploy_keys_projects if remove_remaining_elements
+
+        success
+      end
+    end
+
+    private
+
+    def move_deploy_keys_projects
+      prepare_relation(non_existent_deploy_keys_projects)
+        .update_all(project_id: @project.id)
+    end
+
+    def non_existent_deploy_keys_projects
+      source_project.deploy_keys_projects
+                    .joins(:deploy_key)
+                    .where.not(keys: { fingerprint: @project.deploy_keys.select(:fingerprint) })
+    end
+
+    def remove_remaining_deploy_keys_projects
+      source_project.deploy_keys_projects.destroy_all
+    end
+  end
+end
diff --git a/app/services/projects/move_forks_service.rb b/app/services/projects/move_forks_service.rb
new file mode 100644
index 00000000000..d2901ea1457
--- /dev/null
+++ b/app/services/projects/move_forks_service.rb
@@ -0,0 +1,42 @@
+module Projects
+  class MoveForksService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super && source_project.fork_network
+
+      Project.transaction(requires_new: true) do
+        move_forked_project_links
+        move_fork_network_members
+        update_root_project
+        refresh_forks_count
+
+        success
+      end
+    end
+
+    private
+
+    def move_forked_project_links
+      # Update ancestor
+      ForkedProjectLink.where(forked_to_project: source_project)
+                       .update_all(forked_to_project_id: @project.id)
+
+      # Update the descendants
+      ForkedProjectLink.where(forked_from_project: source_project)
+                       .update_all(forked_from_project_id: @project.id)
+    end
+
+    def move_fork_network_members
+      ForkNetworkMember.where(project: source_project).update_all(project_id: @project.id)
+      ForkNetworkMember.where(forked_from_project: source_project).update_all(forked_from_project_id: @project.id)
+    end
+
+    def update_root_project
+      # Update root network project
+      ForkNetwork.where(root_project: source_project).update_all(root_project_id: @project.id)
+    end
+
+    def refresh_forks_count
+      Projects::ForksCountService.new(@project).refresh_cache
+    end
+  end
+end
diff --git a/app/services/projects/move_lfs_objects_projects_service.rb b/app/services/projects/move_lfs_objects_projects_service.rb
new file mode 100644
index 00000000000..298da5f1a82
--- /dev/null
+++ b/app/services/projects/move_lfs_objects_projects_service.rb
@@ -0,0 +1,29 @@
+module Projects
+  class MoveLfsObjectsProjectsService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      Project.transaction(requires_new: true) do
+        move_lfs_objects_projects
+        remove_remaining_lfs_objects_project if remove_remaining_elements
+
+        success
+      end
+    end
+
+    private
+
+    def move_lfs_objects_projects
+      prepare_relation(non_existent_lfs_objects_projects)
+        .update_all(project_id: @project.lfs_storage_project.id)
+    end
+
+    def remove_remaining_lfs_objects_project
+      source_project.lfs_objects_projects.destroy_all
+    end
+
+    def non_existent_lfs_objects_projects
+      source_project.lfs_objects_projects.where.not(lfs_object: @project.lfs_objects)
+    end
+  end
+end
diff --git a/app/services/projects/move_notification_settings_service.rb b/app/services/projects/move_notification_settings_service.rb
new file mode 100644
index 00000000000..f7be461a5da
--- /dev/null
+++ b/app/services/projects/move_notification_settings_service.rb
@@ -0,0 +1,38 @@
+module Projects
+  class MoveNotificationSettingsService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      Project.transaction(requires_new: true) do
+        move_notification_settings
+        remove_remaining_notification_settings if remove_remaining_elements
+
+        success
+      end
+    end
+
+    private
+
+    def move_notification_settings
+      prepare_relation(non_existent_notifications)
+        .update_all(source_id: @project.id)
+    end
+
+    # Remove remaining notification settings from source_project
+    def remove_remaining_notification_settings
+      source_project.notification_settings.destroy_all
+    end
+
+    # Get users of current notification_settings
+    def users_in_target_project
+      @project.notification_settings.select(:user_id)
+    end
+
+    # Look for notification_settings in source_project that are not in the target project
+    def non_existent_notifications
+      source_project.notification_settings
+        .select(:id)
+        .where.not(user_id: users_in_target_project)
+    end
+  end
+end
diff --git a/app/services/projects/move_project_authorizations_service.rb b/app/services/projects/move_project_authorizations_service.rb
new file mode 100644
index 00000000000..5ef12fc49e5
--- /dev/null
+++ b/app/services/projects/move_project_authorizations_service.rb
@@ -0,0 +1,40 @@
+# NOTE: This service cannot be used directly because it is part of a
+# a bigger process. Instead, use the service MoveAccessService which moves
+# project memberships, project group links, authorizations and refreshes
+# the authorizations if neccessary
+module Projects
+  class MoveProjectAuthorizationsService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      Project.transaction(requires_new: true) do
+        move_project_authorizations
+
+        remove_remaining_authorizations if remove_remaining_elements
+
+        success
+      end
+    end
+
+    private
+
+    def move_project_authorizations
+      prepare_relation(non_existent_authorization, :user_id)
+        .update_all(project_id: @project.id)
+    end
+
+    def remove_remaining_authorizations
+      # I think because the Project Authorization table does not have a primary key
+      # it brings a lot a problems/bugs. First, Rails raises PG::SyntaxException if we use
+      # destroy_all instead of delete_all.
+      source_project.project_authorizations.delete_all(:delete_all)
+    end
+
+    # Look for authorizations in source_project that are not in the target project
+    def non_existent_authorization
+      source_project.project_authorizations
+                    .select(:user_id)
+                    .where.not(user: @project.authorized_users)
+    end
+  end
+end
diff --git a/app/services/projects/move_project_group_links_service.rb b/app/services/projects/move_project_group_links_service.rb
new file mode 100644
index 00000000000..dbeffd7dae9
--- /dev/null
+++ b/app/services/projects/move_project_group_links_service.rb
@@ -0,0 +1,40 @@
+# NOTE: This service cannot be used directly because it is part of a
+# a bigger process. Instead, use the service MoveAccessService which moves
+# project memberships, project group links, authorizations and refreshes
+# the authorizations if neccessary
+module Projects
+  class MoveProjectGroupLinksService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      Project.transaction(requires_new: true) do
+        move_group_links
+        remove_remaining_project_group_links if remove_remaining_elements
+
+        success
+      end
+    end
+
+    private
+
+    def move_group_links
+      prepare_relation(non_existent_group_links)
+        .update_all(project_id: @project.id)
+    end
+
+    # Remove remaining project group links from source_project
+    def remove_remaining_project_group_links
+      source_project.reload.project_group_links.destroy_all
+    end
+
+    def group_links_in_target_project
+      @project.project_group_links.select(:group_id)
+    end
+
+    # Look for groups in source_project that are not in the target project
+    def non_existent_group_links
+      source_project.project_group_links
+                    .where.not(group_id: group_links_in_target_project)
+    end
+  end
+end
diff --git a/app/services/projects/move_project_members_service.rb b/app/services/projects/move_project_members_service.rb
new file mode 100644
index 00000000000..22a5f0a3fe6
--- /dev/null
+++ b/app/services/projects/move_project_members_service.rb
@@ -0,0 +1,40 @@
+# NOTE: This service cannot be used directly because it is part of a
+# a bigger process. Instead, use the service MoveAccessService which moves
+# project memberships, project group links, authorizations and refreshes
+# the authorizations if neccessary
+module Projects
+  class MoveProjectMembersService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      Project.transaction(requires_new: true) do
+        move_project_members
+        remove_remaining_members if remove_remaining_elements
+
+        success
+      end
+    end
+
+    private
+
+    def move_project_members
+      prepare_relation(non_existent_members).update_all(source_id: @project.id)
+    end
+
+    def remove_remaining_members
+      # Remove remaining members and authorizations from source_project
+      source_project.project_members.destroy_all
+    end
+
+    def project_members_in_target_project
+      @project.project_members.select(:user_id)
+    end
+
+    # Look for members in source_project that are not in the target project
+    def non_existent_members
+      source_project.members
+                    .select(:id)
+                    .where.not(user_id: @project.project_members.select(:user_id))
+    end
+  end
+end
diff --git a/app/services/projects/move_users_star_projects_service.rb b/app/services/projects/move_users_star_projects_service.rb
new file mode 100644
index 00000000000..079fd5b9685
--- /dev/null
+++ b/app/services/projects/move_users_star_projects_service.rb
@@ -0,0 +1,20 @@
+module Projects
+  class MoveUsersStarProjectsService < BaseMoveRelationsService
+    def execute(source_project, remove_remaining_elements: true)
+      return unless super
+
+      user_stars = source_project.users_star_projects
+
+      return unless user_stars.any?
+
+      Project.transaction(requires_new: true) do
+        user_stars.update_all(project_id: @project.id)
+
+        Project.reset_counters @project.id, :users_star_projects
+        Project.reset_counters source_project.id, :users_star_projects
+
+        success
+      end
+    end
+  end
+end
diff --git a/app/services/projects/overwrite_project_service.rb b/app/services/projects/overwrite_project_service.rb
new file mode 100644
index 00000000000..ce94f147aa9
--- /dev/null
+++ b/app/services/projects/overwrite_project_service.rb
@@ -0,0 +1,69 @@
+module Projects
+  class OverwriteProjectService < BaseService
+    def execute(source_project)
+      return unless source_project && source_project.namespace == @project.namespace
+
+      Project.transaction do
+        move_before_destroy_relationships(source_project)
+        destroy_old_project(source_project)
+        rename_project(source_project.name, source_project.path)
+
+        @project
+      end
+    # Projects::DestroyService can raise Exceptions, but we don't want
+    # to pass that kind of exception to the caller. Instead, we change it
+    # for a StandardError exception
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      attempt_restore_repositories(source_project)
+
+      if e.class == Exception
+        raise StandardError, e.message
+      else
+        raise
+      end
+    end
+
+    private
+
+    def move_before_destroy_relationships(source_project)
+      options = { remove_remaining_elements: false }
+
+      ::Projects::MoveUsersStarProjectsService.new(@project, @current_user).execute(source_project, options)
+      ::Projects::MoveAccessService.new(@project, @current_user).execute(source_project, options)
+      ::Projects::MoveDeployKeysProjectsService.new(@project, @current_user).execute(source_project, options)
+      ::Projects::MoveNotificationSettingsService.new(@project, @current_user).execute(source_project, options)
+      ::Projects::MoveForksService.new(@project, @current_user).execute(source_project, options)
+      ::Projects::MoveLfsObjectsProjectsService.new(@project, @current_user).execute(source_project, options)
+      add_source_project_to_fork_network(source_project)
+    end
+
+    def destroy_old_project(source_project)
+      # Delete previous project (synchronously) and unlink relations
+      ::Projects::DestroyService.new(source_project, @current_user).execute
+    end
+
+    def rename_project(name, path)
+      # Update de project's name and path to the original name/path
+      ::Projects::UpdateService.new(@project,
+                                    @current_user,
+                                    { name: name, path: path })
+                               .execute
+    end
+
+    def attempt_restore_repositories(project)
+      ::Projects::DestroyService.new(project, @current_user).attempt_repositories_rollback
+    end
+
+    def add_source_project_to_fork_network(source_project)
+      return unless @project.fork_network
+
+      # Because he have moved all references in the fork network from the source_project
+      # we won't be able to query the database (only through its cached data),
+      # for its former relationships. That's why we're adding it to the network
+      # as a fork of the target project
+      ForkNetworkMember.create!(fork_network: @project.fork_network,
+                                project: source_project,
+                                forked_from_project: @project)
+    end
+  end
+end