Rename namespace-paths in a migration helper

5 files changed, 469 insertions, 0 deletions

diff --git a/lib/gitlab/database/rename_reserved_paths_migration.rb b/lib/gitlab/database/rename_reserved_paths_migration.rb
new file mode 100644
index 00000000000..ad5570b4c72
--- /dev/null
+++ b/lib/gitlab/database/rename_reserved_paths_migration.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module Database
+    module RenameReservedPathsMigration
+      include MigrationHelpers
+      include Namespaces
+      include Projects
+
+      def rename_wildcard_paths(one_or_more_paths)
+        paths = Array(one_or_more_paths)
+        rename_namespaces(paths, type: :wildcard)
+      end
+
+      def rename_root_paths(paths)
+        paths = Array(paths)
+        rename_namespaces(paths, type: :top_level)
+      end
+
+      def rename_path(namespace_path, path_was)
+        counter = 0
+        path = "#{path_was}#{counter}"
+
+        while route_exists?(File.join(namespace_path, path))
+          counter += 1
+          path = "#{path_was}#{counter}"
+        end
+
+        path
+      end
+
+      def route_exists?(full_path)
+        MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/migration_classes.rb
new file mode 100644
index 00000000000..a919d250541
--- /dev/null
+++ b/lib/gitlab/database/rename_reserved_paths_migration/migration_classes.rb
@@ -0,0 +1,84 @@
+module Gitlab
+  module Database
+    module RenameReservedPathsMigration
+      module MigrationClasses
+        class User < ActiveRecord::Base
+          self.table_name = 'users'
+        end
+
+        class Namespace < ActiveRecord::Base
+          self.table_name = 'namespaces'
+          belongs_to :parent,
+                     class_name: "#{MigrationClasses.name}::Namespace"
+          has_one :route, as: :source
+          has_many :children,
+                   class_name: "#{MigrationClasses.name}::Namespace",
+                   foreign_key: :parent_id
+          belongs_to :owner,
+                     class_name: "#{MigrationClasses.name}::User"
+
+          # Overridden to have the correct `source_type` for the `route` relation
+          def self.name
+            'Namespace'
+          end
+
+          def full_path
+            if route && route.path.present?
+              @full_path ||= route.path
+            else
+              update_route if persisted?
+
+              build_full_path
+            end
+          end
+
+          def build_full_path
+            if parent && path
+              parent.full_path + '/' + path
+            else
+              path
+            end
+          end
+
+          def update_route
+            prepare_route
+            route.save
+          end
+
+          def prepare_route
+            route || build_route(source: self)
+            route.path = build_full_path
+            route.name = build_full_name
+            @full_path = nil
+            @full_name = nil
+          end
+
+          def build_full_name
+            if parent && name
+              parent.human_name + ' / ' + name
+            else
+              name
+            end
+          end
+
+          def human_name
+            owner&.name
+          end
+        end
+
+        class Route < ActiveRecord::Base
+          self.table_name = 'routes'
+          belongs_to :source, polymorphic: true
+        end
+
+        class Project < ActiveRecord::Base
+          self.table_name = 'projects'
+
+          def repository_storage_path
+            Gitlab.config.repositories.storages[repository_storage]['path']
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/namespaces.rb
new file mode 100644
index 00000000000..f8fbeaa990a
--- /dev/null
+++ b/lib/gitlab/database/rename_reserved_paths_migration/namespaces.rb
@@ -0,0 +1,113 @@
+module Gitlab
+  module Database
+    module RenameReservedPathsMigration
+      module Namespaces
+        include Gitlab::ShellAdapter
+
+        def rename_namespaces(paths, type:)
+          namespaces_for_paths(paths, type: type).each do |namespace|
+            rename_namespace(namespace)
+          end
+        end
+
+        def namespaces_for_paths(paths, type:)
+          namespaces = if type == :wildcard
+                         MigrationClasses::Namespace.where.not(parent_id: nil)
+                       elsif type == :top_level
+                         MigrationClasses::Namespace.where(parent_id: nil)
+                       end
+          namespaces.where(path: paths.map(&:downcase))
+        end
+
+        def rename_namespace(namespace)
+          old_path = namespace.path
+          old_full_path = namespace.full_path
+          # Only remove the last occurrence of the path name to get the parent namespace path
+          namespace_path = remove_last_occurrence(old_full_path, old_path)
+          new_path = rename_path(namespace_path, old_path)
+          new_full_path = if namespace_path.present?
+                            File.join(namespace_path, new_path)
+                          else
+                            new_path
+                          end
+
+          # skips callbacks & validations
+          MigrationClasses::Namespace.where(id: namespace).
+            update_all(path: new_path)
+
+          replace_statement = replace_sql(Route.arel_table[:path],
+                                          old_full_path,
+                                          new_full_path)
+
+          update_column_in_batches(:routes, :path, replace_statement)  do |table, query|
+            query.where(MigrationClasses::Route.arel_table[:path].matches("#{old_full_path}%"))
+          end
+
+          move_repositories(namespace, old_full_path, new_full_path)
+          move_namespace_folders(uploads_dir, old_full_path, new_full_path) if file_storage?
+          move_namespace_folders(pages_dir, old_full_path, new_full_path)
+        end
+
+        def move_namespace_folders(directory, old_relative_path, new_relative_path)
+          old_path = File.join(directory, old_relative_path)
+          return unless File.directory?(old_path)
+
+          new_path = File.join(directory, new_relative_path)
+          FileUtils.mv(old_path, new_path)
+        end
+
+        def move_repositories(namespace, old_full_path, new_full_path)
+          repo_paths_for_namespace(namespace).each do |repository_storage_path|
+            # Ensure old directory exists before moving it
+            gitlab_shell.add_namespace(repository_storage_path, old_full_path)
+
+            unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
+              message = "Exception moving path #{repository_storage_path} \
+                           from #{old_full_path} to #{new_full_path}"
+              Rails.logger.error message
+            end
+          end
+        end
+
+        def repo_paths_for_namespace(namespace)
+          projects_for_namespace(namespace).
+            select('distinct(repository_storage)').map(&:repository_storage_path)
+        end
+
+        def projects_for_namespace(namespace)
+          namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id])
+          namespace_or_children = MigrationClasses::Project.
+                                    arel_table[:namespace_id].
+                                    in(namespace_ids)
+          MigrationClasses::Project.unscoped.where(namespace_or_children)
+        end
+
+        # This won't scale to huge trees, but it should do for a handful of
+        # namespaces called `system`.
+        def child_ids_for_parent(namespace, ids: [])
+          namespace.children.each do |child|
+            ids << child.id
+            child_ids_for_parent(child, ids: ids) if child.children.any?
+          end
+          ids
+        end
+
+        def remove_last_occurrence(string, pattern)
+          string.reverse.sub(pattern.reverse, "").reverse
+        end
+
+        def file_storage?
+          CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
+        end
+
+        def uploads_dir
+          File.join(CarrierWave.root, "uploads")
+        end
+
+        def pages_dir
+          Settings.pages.path
+        end
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/namespaces_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/namespaces_spec.rb
new file mode 100644
index 00000000000..ea31e373647
--- /dev/null
+++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/namespaces_spec.rb
@@ -0,0 +1,208 @@
+require 'spec_helper'
+
+describe Gitlab::Database::RenameReservedPathsMigration::Namespaces, :truncate do
+  let(:test_dir) { File.join(Rails.root, 'tmp', 'tests', 'rename_namespaces_test') }
+  let(:uploads_dir) { File.join(test_dir, 'public', 'uploads') }
+  let(:subject) do
+    ActiveRecord::Migration.new.extend(
+      Gitlab::Database::RenameReservedPathsMigration
+    )
+  end
+
+  before do
+    FileUtils.remove_dir(test_dir) if File.directory?(test_dir)
+    FileUtils.mkdir_p(uploads_dir)
+    FileUtils.remove_dir(TestEnv.repos_path) if File.directory?(TestEnv.repos_path)
+    allow(subject).to receive(:uploads_dir).and_return(uploads_dir)
+    allow(subject).to receive(:say)
+  end
+
+  def migration_namespace(namespace)
+    Gitlab::Database::RenameReservedPathsMigration::MigrationClasses::
+      Namespace.find(namespace.id)
+  end
+
+  describe '#namespaces_for_paths' do
+    context 'for wildcard namespaces' do
+      it 'only returns child namespaces with the correct path' do
+        _root_namespace = create(:namespace, path: 'the-path')
+        _other_path = create(:namespace,
+                             path: 'other',
+                             parent: create(:namespace))
+        namespace = create(:namespace,
+                           path: 'the-path',
+                           parent: create(:namespace))
+
+        found_ids = subject.namespaces_for_paths(['the-path'], type: :wildcard).
+                      pluck(:id)
+        expect(found_ids).to contain_exactly(namespace.id)
+      end
+    end
+
+    context 'for top level namespaces' do
+      it 'only returns child namespaces with the correct path' do
+        root_namespace = create(:namespace, path: 'the-path')
+        _other_path = create(:namespace, path: 'other')
+        _child_namespace = create(:namespace,
+                           path: 'the-path',
+                           parent: create(:namespace))
+
+        found_ids = subject.namespaces_for_paths(['the-path'], type: :top_level).
+                      pluck(:id)
+        expect(found_ids).to contain_exactly(root_namespace.id)
+      end
+    end
+  end
+
+  describe '#move_repositories' do
+    let(:namespace) { create(:group, name: 'hello-group') }
+    it 'moves a project for a namespace' do
+      create(:project, namespace: namespace, path: 'hello-project')
+      expected_path = File.join(TestEnv.repos_path, 'bye-group', 'hello-project.git')
+
+      subject.move_repositories(namespace, 'hello-group', 'bye-group')
+
+      expect(File.directory?(expected_path)).to be(true)
+    end
+
+    it 'moves a namespace in a subdirectory correctly' do
+      child_namespace = create(:group, name: 'sub-group', parent: namespace)
+      create(:project, namespace: child_namespace, path: 'hello-project')
+
+      expected_path = File.join(TestEnv.repos_path, 'hello-group', 'renamed-sub-group', 'hello-project.git')
+
+      subject.move_repositories(child_namespace, 'hello-group/sub-group', 'hello-group/renamed-sub-group')
+
+      expect(File.directory?(expected_path)).to be(true)
+    end
+
+    it 'moves a parent namespace with subdirectories' do
+      child_namespace = create(:group, name: 'sub-group', parent: namespace)
+      create(:project, namespace: child_namespace, path: 'hello-project')
+      expected_path = File.join(TestEnv.repos_path, 'renamed-group', 'sub-group', 'hello-project.git')
+
+      subject.move_repositories(child_namespace, 'hello-group', 'renamed-group')
+
+      expect(File.directory?(expected_path)).to be(true)
+    end
+  end
+
+  describe '#move_namespace_folders' do
+    it 'moves a namespace with files' do
+      source = File.join(uploads_dir, 'parent-group', 'sub-group')
+      FileUtils.mkdir_p(source)
+      destination = File.join(uploads_dir, 'parent-group', 'moved-group')
+      FileUtils.touch(File.join(source, 'test.txt'))
+      expected_file = File.join(destination, 'test.txt')
+
+      subject.move_namespace_folders(uploads_dir, File.join('parent-group', 'sub-group'), File.join('parent-group', 'moved-group'))
+
+      expect(File.exist?(expected_file)).to be(true)
+    end
+
+    it 'moves a parent namespace uploads' do
+      source = File.join(uploads_dir, 'parent-group', 'sub-group')
+      FileUtils.mkdir_p(source)
+      destination = File.join(uploads_dir, 'moved-parent', 'sub-group')
+      FileUtils.touch(File.join(source, 'test.txt'))
+      expected_file = File.join(destination, 'test.txt')
+
+      subject.move_namespace_folders(uploads_dir, 'parent-group', 'moved-parent')
+
+      expect(File.exist?(expected_file)).to be(true)
+    end
+  end
+
+  describe "#child_ids_for_parent" do
+    it "collects child ids for all levels" do
+      parent = create(:namespace)
+      first_child = create(:namespace, parent: parent)
+      second_child = create(:namespace, parent: parent)
+      third_child = create(:namespace, parent: second_child)
+      all_ids = [parent.id, first_child.id, second_child.id, third_child.id]
+
+      collected_ids = subject.child_ids_for_parent(parent, ids: [parent.id])
+
+      expect(collected_ids).to contain_exactly(*all_ids)
+    end
+  end
+
+  describe "#remove_last_ocurrence" do
+    it "removes only the last occurance of a string" do
+      input = "this/is/a-word-to-replace/namespace/with/a-word-to-replace"
+
+      expect(subject.remove_last_occurrence(input, "a-word-to-replace"))
+        .to eq("this/is/a-word-to-replace/namespace/with/")
+    end
+  end
+
+  describe "#rename_namespace" do
+    let(:namespace) { create(:namespace, path: 'the-path') }
+    it "renames namespaces called the-path" do
+      subject.rename_namespace(namespace)
+
+      expect(namespace.reload.path).to eq("the-path0")
+    end
+
+    it "renames the route to the namespace" do
+      subject.rename_namespace(namespace)
+
+      expect(Namespace.find(namespace.id).full_path).to eq("the-path0")
+    end
+
+    it "renames the route for projects of the namespace" do
+      project = create(:project, path: "project-path", namespace: namespace)
+
+      subject.rename_namespace(namespace)
+
+      expect(project.route.reload.path).to eq("the-path0/project-path")
+    end
+
+    it "moves the the repository for a project in the namespace" do
+      create(:project, namespace: namespace, path: "the-path-project")
+      expected_repo = File.join(TestEnv.repos_path, "the-path0", "the-path-project.git")
+
+      subject.rename_namespace(namespace)
+
+      expect(File.directory?(expected_repo)).to be(true)
+    end
+
+    it "moves the uploads for the namespace" do
+      allow(subject).to receive(:move_namespace_folders).with(Settings.pages.path, "the-path", "the-path0")
+      expect(subject).to receive(:move_namespace_folders).with(uploads_dir, "the-path", "the-path0")
+
+      subject.rename_namespace(namespace)
+    end
+
+    it "moves the pages for the namespace" do
+      allow(subject).to receive(:move_namespace_folders).with(uploads_dir, "the-path", "the-path0")
+      expect(subject).to receive(:move_namespace_folders).with(Settings.pages.path, "the-path", "the-path0")
+
+      subject.rename_namespace(namespace)
+    end
+
+    context "the-path namespace -> subgroup -> the-path0 project" do
+      it "updates the route of the project correctly" do
+        subgroup = create(:group, path: "subgroup", parent: namespace)
+        project = create(:project, path: "the-path0", namespace: subgroup)
+
+        subject.rename_namespace(namespace)
+
+        expect(project.route.reload.path).to eq("the-path0/subgroup/the-path0")
+      end
+    end
+  end
+
+  describe '#rename_namespaces' do
+    context 'top level namespaces' do
+      let!(:namespace) { create(:namespace, path: 'the-path') }
+
+      it 'should rename the namespace' do
+        expect(subject).to receive(:rename_namespace).
+                             with(migration_namespace(namespace))
+
+        subject.rename_namespaces(['the-path'], type: :top_level)
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration_spec.rb
new file mode 100644
index 00000000000..8b4ab6703c6
--- /dev/null
+++ b/spec/lib/gitlab/database/rename_reserved_paths_migration_spec.rb
@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe Gitlab::Database::RenameReservedPathsMigration do
+  let(:subject) do
+    ActiveRecord::Migration.new.extend(
+      Gitlab::Database::RenameReservedPathsMigration
+    )
+  end
+
+  describe '#rename_wildcard_paths' do
+    it 'should rename namespaces' do
+      expect(subject).to receive(:rename_namespaces).
+                           with(['first-path', 'second-path'], type: :wildcard)
+
+      subject.rename_wildcard_paths(['first-path', 'second-path'])
+    end
+
+    it 'should rename projects'
+  end
+
+  describe '#rename_root_paths' do
+    it 'should rename namespaces' do
+      expect(subject).to receive(:rename_namespaces).
+                           with(['the-path'], type: :top_level)
+
+      subject.rename_root_paths('the-path')
+    end
+  end
+end