Merge branch 'bvl-rename-all-reserved-paths' into 'master'

Rename all forbidden paths again

Closes #32625

See merge request !11713

4 files changed, 139 insertions, 20 deletions

diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
index 89530082cd2..f333ff22300 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
@@ -29,6 +29,11 @@ module Gitlab
           paths = Array(paths)
           RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level)
         end
+
+        def revert_renames
+          RenameProjects.new([], self).revert_renames
+          RenameNamespaces.new([], self).revert_renames
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
index d8163d7da11..33f8939bc61 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
@@ -6,7 +6,10 @@ module Gitlab
           attr_reader :paths, :migration
 
           delegate :update_column_in_batches,
+                   :execute,
                    :replace_sql,
+                   :quote_string,
+                   :say,
                    to: :migration
 
           def initialize(paths, migration)
@@ -26,24 +29,45 @@ module Gitlab
             new_path = rename_path(namespace_path, old_path)
             new_full_path = join_routable_path(namespace_path, new_path)
 
+            perform_rename(routable, old_full_path, new_full_path)
+
+            [old_full_path, new_full_path]
+          end
+
+          def perform_rename(routable, old_full_path, new_full_path)
             # skips callbacks & validations
+            new_path = new_full_path.split('/').last
             routable.class.where(id: routable)
               .update_all(path: new_path)
 
             rename_routes(old_full_path, new_full_path)
-
-            [old_full_path, new_full_path]
           end
 
           def rename_routes(old_full_path, new_full_path)
+            routes = Route.arel_table
+
+            quoted_old_full_path = quote_string(old_full_path)
+            quoted_old_wildcard_path = quote_string("#{old_full_path}/%")
+
+            filter = if Database.mysql?
+                       "lower(routes.path) = lower('#{quoted_old_full_path}') "\
+                       "OR routes.path LIKE '#{quoted_old_wildcard_path}'"
+                     else
+                       "routes.id IN "\
+                       "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\
+                       "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )"
+                     end
+
             replace_statement = replace_sql(Route.arel_table[:path],
                                             old_full_path,
                                             new_full_path)
 
-            update_column_in_batches(:routes, :path, replace_statement)  do |table, query|
-              path_or_children = table[:path].matches_any([old_full_path, "#{old_full_path}/%"])
-              query.where(path_or_children)
-            end
+            update = Arel::UpdateManager.new(ActiveRecord::Base)
+                       .table(routes)
+                       .set([[routes[:path], replace_statement]])
+                       .where(Arel::Nodes::SqlLiteral.new(filter))
+
+            execute(update.to_sql)
           end
 
           def rename_path(namespace_path, path_was)
@@ -86,32 +110,74 @@ module Gitlab
 
           def move_folders(directory, old_relative_path, new_relative_path)
             old_path = File.join(directory, old_relative_path)
-            return unless File.directory?(old_path)
+            unless File.directory?(old_path)
+              say "#{old_path} doesn't exist, skipping"
+              return
+            end
 
             new_path = File.join(directory, new_relative_path)
             FileUtils.mv(old_path, new_path)
           end
 
           def remove_cached_html_for_projects(project_ids)
-            update_column_in_batches(:projects, :description_html, nil) do |table, query|
-              query.where(table[:id].in(project_ids))
-            end
-
-            update_column_in_batches(:issues, :description_html, nil) do |table, query|
-              query.where(table[:project_id].in(project_ids))
+            project_ids.each do |project_id|
+              update_column_in_batches(:projects, :description_html, nil) do |table, query|
+                query.where(table[:id].eq(project_id))
+              end
+
+              update_column_in_batches(:issues, :description_html, nil) do |table, query|
+                query.where(table[:project_id].eq(project_id))
+              end
+
+              update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
+                query.where(table[:target_project_id].eq(project_id))
+              end
+
+              update_column_in_batches(:notes, :note_html, nil) do |table, query|
+                query.where(table[:project_id].eq(project_id))
+              end
+
+              update_column_in_batches(:milestones, :description_html, nil) do |table, query|
+                query.where(table[:project_id].eq(project_id))
+              end
             end
+          end
 
-            update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
-              query.where(table[:target_project_id].in(project_ids))
+          def track_rename(type, old_path, new_path)
+            key = redis_key_for_type(type)
+            Gitlab::Redis.with do |redis|
+              redis.lpush(key, [old_path, new_path].to_json)
+              redis.expire(key, 2.weeks.to_i)
             end
+            say "tracked rename: #{key}: #{old_path} -> #{new_path}"
+          end
 
-            update_column_in_batches(:notes, :note_html, nil) do |table, query|
-              query.where(table[:project_id].in(project_ids))
+          def reverts_for_type(type)
+            key = redis_key_for_type(type)
+
+            Gitlab::Redis.with do |redis|
+              failed_reverts = []
+
+              while rename_info = redis.lpop(key)
+                path_before_rename, path_after_rename = JSON.parse(rename_info)
+                say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}"
+                begin
+                  yield(path_before_rename, path_after_rename)
+                rescue StandardError => e
+                  failed_reverts << rename_info
+                  say "Renaming #{type} from #{path_after_rename} back to "\
+                      "#{path_before_rename} failed. Review the error and try "\
+                      "again by running the `down` action. \n"\
+                      "#{e.message}: \n #{e.backtrace.join("\n")}"
+                end
+              end
+
+              failed_reverts.each { |rename_info| redis.lpush(key, rename_info) }
             end
+          end
 
-            update_column_in_batches(:milestones, :description_html, nil) do |table, query|
-              query.where(table[:project_id].in(project_ids))
-            end
+          def redis_key_for_type(type)
+            "rename:#{migration.name}:#{type}"
           end
 
           def file_storage?
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
index da7e2cb2e85..05b86f32ce2 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
@@ -26,6 +26,12 @@ module Gitlab
           def rename_namespace(namespace)
             old_full_path, new_full_path = rename_path_for_routable(namespace)
 
+            track_rename('namespace', old_full_path, new_full_path)
+
+            rename_namespace_dependencies(namespace, old_full_path, new_full_path)
+          end
+
+          def rename_namespace_dependencies(namespace, old_full_path, new_full_path)
             move_repositories(namespace, old_full_path, new_full_path)
             move_uploads(old_full_path, new_full_path)
             move_pages(old_full_path, new_full_path)
@@ -33,6 +39,23 @@ module Gitlab
             remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id))
           end
 
+          def revert_renames
+            reverts_for_type('namespace') do |path_before_rename, current_path|
+              matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
+              namespace = MigrationClasses::Namespace.joins(:route)
+                            .where(matches_path).first&.becomes(MigrationClasses::Namespace)
+
+              if namespace
+                perform_rename(namespace, current_path, path_before_rename)
+
+                rename_namespace_dependencies(namespace, current_path, path_before_rename)
+              else
+                say "Couldn't rename namespace from #{current_path} back to #{path_before_rename}, "\
+                    "namespace was renamed, or no longer exists at the expected path"
+              end
+            end
+          end
+
           def rename_user(old_username, new_username)
             MigrationClasses::User.where(username: old_username)
               .update_all(username: new_username)
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
index 448717eb744..75a75f61953 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
@@ -16,12 +16,37 @@ module Gitlab
           def rename_project(project)
             old_full_path, new_full_path = rename_path_for_routable(project)
 
+            track_rename('project', old_full_path, new_full_path)
+
+            move_project_folders(project, old_full_path, new_full_path)
+          end
+
+          def move_project_folders(project, old_full_path, new_full_path)
             move_repository(project, old_full_path, new_full_path)
             move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki")
             move_uploads(old_full_path, new_full_path)
             move_pages(old_full_path, new_full_path)
           end
 
+          def revert_renames
+            reverts_for_type('project') do |path_before_rename, current_path|
+              matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
+              project = MigrationClasses::Project.joins(:route)
+                          .where(matches_path).first
+
+              if project
+                perform_rename(project, current_path, path_before_rename)
+
+                move_project_folders(project, current_path, path_before_rename)
+              else
+                say "Couldn't rename project from #{current_path} back to "\
+                    "#{path_before_rename}, project was renamed or no longer "\
+                    "exists at the expected path."
+
+              end
+            end
+          end
+
           def move_repository(project, old_path, new_path)
             unless gitlab_shell.mv_repository(project.repository_storage_path,
                                               old_path,