diff options
5 files changed, 469 insertions, 0 deletions
diff --git a/lib/gitlab/database/rename_reserved_paths_migration.rb b/lib/gitlab/database/rename_reserved_paths_migration.rb new file mode 100644 index 00000000000..ad5570b4c72 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration.rb @@ -0,0 +1,35 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + include MigrationHelpers + include Namespaces + include Projects + + def rename_wildcard_paths(one_or_more_paths) + paths = Array(one_or_more_paths) + rename_namespaces(paths, type: :wildcard) + end + + def rename_root_paths(paths) + paths = Array(paths) + rename_namespaces(paths, type: :top_level) + end + + def rename_path(namespace_path, path_was) + counter = 0 + path = "#{path_was}#{counter}" + + while route_exists?(File.join(namespace_path, path)) + counter += 1 + path = "#{path_was}#{counter}" + end + + path + end + + def route_exists?(full_path) + MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any? + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/migration_classes.rb new file mode 100644 index 00000000000..a919d250541 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/migration_classes.rb @@ -0,0 +1,84 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module MigrationClasses + class User < ActiveRecord::Base + self.table_name = 'users' + end + + class Namespace < ActiveRecord::Base + self.table_name = 'namespaces' + belongs_to :parent, + class_name: "#{MigrationClasses.name}::Namespace" + has_one :route, as: :source + has_many :children, + class_name: "#{MigrationClasses.name}::Namespace", + foreign_key: :parent_id + belongs_to :owner, + class_name: "#{MigrationClasses.name}::User" + + # Overridden to have the correct `source_type` for the `route` relation + def self.name + 'Namespace' + end + + def full_path + if route && route.path.present? + @full_path ||= route.path + else + update_route if persisted? + + build_full_path + end + end + + def build_full_path + if parent && path + parent.full_path + '/' + path + else + path + end + end + + def update_route + prepare_route + route.save + end + + def prepare_route + route || build_route(source: self) + route.path = build_full_path + route.name = build_full_name + @full_path = nil + @full_name = nil + end + + def build_full_name + if parent && name + parent.human_name + ' / ' + name + else + name + end + end + + def human_name + owner&.name + end + end + + class Route < ActiveRecord::Base + self.table_name = 'routes' + belongs_to :source, polymorphic: true + end + + class Project < ActiveRecord::Base + self.table_name = 'projects' + + def repository_storage_path + Gitlab.config.repositories.storages[repository_storage]['path'] + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/namespaces.rb new file mode 100644 index 00000000000..f8fbeaa990a --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/namespaces.rb @@ -0,0 +1,113 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module Namespaces + include Gitlab::ShellAdapter + + def rename_namespaces(paths, type:) + namespaces_for_paths(paths, type: type).each do |namespace| + rename_namespace(namespace) + end + end + + def namespaces_for_paths(paths, type:) + namespaces = if type == :wildcard + MigrationClasses::Namespace.where.not(parent_id: nil) + elsif type == :top_level + MigrationClasses::Namespace.where(parent_id: nil) + end + namespaces.where(path: paths.map(&:downcase)) + end + + def rename_namespace(namespace) + old_path = namespace.path + old_full_path = namespace.full_path + # Only remove the last occurrence of the path name to get the parent namespace path + namespace_path = remove_last_occurrence(old_full_path, old_path) + new_path = rename_path(namespace_path, old_path) + new_full_path = if namespace_path.present? + File.join(namespace_path, new_path) + else + new_path + end + + # skips callbacks & validations + MigrationClasses::Namespace.where(id: namespace). + update_all(path: new_path) + + replace_statement = replace_sql(Route.arel_table[:path], + old_full_path, + new_full_path) + + update_column_in_batches(:routes, :path, replace_statement) do |table, query| + query.where(MigrationClasses::Route.arel_table[:path].matches("#{old_full_path}%")) + end + + move_repositories(namespace, old_full_path, new_full_path) + move_namespace_folders(uploads_dir, old_full_path, new_full_path) if file_storage? + move_namespace_folders(pages_dir, old_full_path, new_full_path) + end + + def move_namespace_folders(directory, old_relative_path, new_relative_path) + old_path = File.join(directory, old_relative_path) + return unless File.directory?(old_path) + + new_path = File.join(directory, new_relative_path) + FileUtils.mv(old_path, new_path) + end + + def move_repositories(namespace, old_full_path, new_full_path) + repo_paths_for_namespace(namespace).each do |repository_storage_path| + # Ensure old directory exists before moving it + gitlab_shell.add_namespace(repository_storage_path, old_full_path) + + unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path) + message = "Exception moving path #{repository_storage_path} \ + from #{old_full_path} to #{new_full_path}" + Rails.logger.error message + end + end + end + + def repo_paths_for_namespace(namespace) + projects_for_namespace(namespace). + select('distinct(repository_storage)').map(&:repository_storage_path) + end + + def projects_for_namespace(namespace) + namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id]) + namespace_or_children = MigrationClasses::Project. + arel_table[:namespace_id]. + in(namespace_ids) + MigrationClasses::Project.unscoped.where(namespace_or_children) + end + + # This won't scale to huge trees, but it should do for a handful of + # namespaces called `system`. + def child_ids_for_parent(namespace, ids: []) + namespace.children.each do |child| + ids << child.id + child_ids_for_parent(child, ids: ids) if child.children.any? + end + ids + end + + def remove_last_occurrence(string, pattern) + string.reverse.sub(pattern.reverse, "").reverse + end + + def file_storage? + CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File + end + + def uploads_dir + File.join(CarrierWave.root, "uploads") + end + + def pages_dir + Settings.pages.path + end + end + end + end +end diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/namespaces_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/namespaces_spec.rb new file mode 100644 index 00000000000..ea31e373647 --- /dev/null +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/namespaces_spec.rb @@ -0,0 +1,208 @@ +require 'spec_helper' + +describe Gitlab::Database::RenameReservedPathsMigration::Namespaces, :truncate do + let(:test_dir) { File.join(Rails.root, 'tmp', 'tests', 'rename_namespaces_test') } + let(:uploads_dir) { File.join(test_dir, 'public', 'uploads') } + let(:subject) do + ActiveRecord::Migration.new.extend( + Gitlab::Database::RenameReservedPathsMigration + ) + end + + before do + FileUtils.remove_dir(test_dir) if File.directory?(test_dir) + FileUtils.mkdir_p(uploads_dir) + FileUtils.remove_dir(TestEnv.repos_path) if File.directory?(TestEnv.repos_path) + allow(subject).to receive(:uploads_dir).and_return(uploads_dir) + allow(subject).to receive(:say) + end + + def migration_namespace(namespace) + Gitlab::Database::RenameReservedPathsMigration::MigrationClasses:: + Namespace.find(namespace.id) + end + + describe '#namespaces_for_paths' do + context 'for wildcard namespaces' do + it 'only returns child namespaces with the correct path' do + _root_namespace = create(:namespace, path: 'the-path') + _other_path = create(:namespace, + path: 'other', + parent: create(:namespace)) + namespace = create(:namespace, + path: 'the-path', + parent: create(:namespace)) + + found_ids = subject.namespaces_for_paths(['the-path'], type: :wildcard). + pluck(:id) + expect(found_ids).to contain_exactly(namespace.id) + end + end + + context 'for top level namespaces' do + it 'only returns child namespaces with the correct path' do + root_namespace = create(:namespace, path: 'the-path') + _other_path = create(:namespace, path: 'other') + _child_namespace = create(:namespace, + path: 'the-path', + parent: create(:namespace)) + + found_ids = subject.namespaces_for_paths(['the-path'], type: :top_level). + pluck(:id) + expect(found_ids).to contain_exactly(root_namespace.id) + end + end + end + + describe '#move_repositories' do + let(:namespace) { create(:group, name: 'hello-group') } + it 'moves a project for a namespace' do + create(:project, namespace: namespace, path: 'hello-project') + expected_path = File.join(TestEnv.repos_path, 'bye-group', 'hello-project.git') + + subject.move_repositories(namespace, 'hello-group', 'bye-group') + + expect(File.directory?(expected_path)).to be(true) + end + + it 'moves a namespace in a subdirectory correctly' do + child_namespace = create(:group, name: 'sub-group', parent: namespace) + create(:project, namespace: child_namespace, path: 'hello-project') + + expected_path = File.join(TestEnv.repos_path, 'hello-group', 'renamed-sub-group', 'hello-project.git') + + subject.move_repositories(child_namespace, 'hello-group/sub-group', 'hello-group/renamed-sub-group') + + expect(File.directory?(expected_path)).to be(true) + end + + it 'moves a parent namespace with subdirectories' do + child_namespace = create(:group, name: 'sub-group', parent: namespace) + create(:project, namespace: child_namespace, path: 'hello-project') + expected_path = File.join(TestEnv.repos_path, 'renamed-group', 'sub-group', 'hello-project.git') + + subject.move_repositories(child_namespace, 'hello-group', 'renamed-group') + + expect(File.directory?(expected_path)).to be(true) + end + end + + describe '#move_namespace_folders' do + it 'moves a namespace with files' do + source = File.join(uploads_dir, 'parent-group', 'sub-group') + FileUtils.mkdir_p(source) + destination = File.join(uploads_dir, 'parent-group', 'moved-group') + FileUtils.touch(File.join(source, 'test.txt')) + expected_file = File.join(destination, 'test.txt') + + subject.move_namespace_folders(uploads_dir, File.join('parent-group', 'sub-group'), File.join('parent-group', 'moved-group')) + + expect(File.exist?(expected_file)).to be(true) + end + + it 'moves a parent namespace uploads' do + source = File.join(uploads_dir, 'parent-group', 'sub-group') + FileUtils.mkdir_p(source) + destination = File.join(uploads_dir, 'moved-parent', 'sub-group') + FileUtils.touch(File.join(source, 'test.txt')) + expected_file = File.join(destination, 'test.txt') + + subject.move_namespace_folders(uploads_dir, 'parent-group', 'moved-parent') + + expect(File.exist?(expected_file)).to be(true) + end + end + + describe "#child_ids_for_parent" do + it "collects child ids for all levels" do + parent = create(:namespace) + first_child = create(:namespace, parent: parent) + second_child = create(:namespace, parent: parent) + third_child = create(:namespace, parent: second_child) + all_ids = [parent.id, first_child.id, second_child.id, third_child.id] + + collected_ids = subject.child_ids_for_parent(parent, ids: [parent.id]) + + expect(collected_ids).to contain_exactly(*all_ids) + end + end + + describe "#remove_last_ocurrence" do + it "removes only the last occurance of a string" do + input = "this/is/a-word-to-replace/namespace/with/a-word-to-replace" + + expect(subject.remove_last_occurrence(input, "a-word-to-replace")) + .to eq("this/is/a-word-to-replace/namespace/with/") + end + end + + describe "#rename_namespace" do + let(:namespace) { create(:namespace, path: 'the-path') } + it "renames namespaces called the-path" do + subject.rename_namespace(namespace) + + expect(namespace.reload.path).to eq("the-path0") + end + + it "renames the route to the namespace" do + subject.rename_namespace(namespace) + + expect(Namespace.find(namespace.id).full_path).to eq("the-path0") + end + + it "renames the route for projects of the namespace" do + project = create(:project, path: "project-path", namespace: namespace) + + subject.rename_namespace(namespace) + + expect(project.route.reload.path).to eq("the-path0/project-path") + end + + it "moves the the repository for a project in the namespace" do + create(:project, namespace: namespace, path: "the-path-project") + expected_repo = File.join(TestEnv.repos_path, "the-path0", "the-path-project.git") + + subject.rename_namespace(namespace) + + expect(File.directory?(expected_repo)).to be(true) + end + + it "moves the uploads for the namespace" do + allow(subject).to receive(:move_namespace_folders).with(Settings.pages.path, "the-path", "the-path0") + expect(subject).to receive(:move_namespace_folders).with(uploads_dir, "the-path", "the-path0") + + subject.rename_namespace(namespace) + end + + it "moves the pages for the namespace" do + allow(subject).to receive(:move_namespace_folders).with(uploads_dir, "the-path", "the-path0") + expect(subject).to receive(:move_namespace_folders).with(Settings.pages.path, "the-path", "the-path0") + + subject.rename_namespace(namespace) + end + + context "the-path namespace -> subgroup -> the-path0 project" do + it "updates the route of the project correctly" do + subgroup = create(:group, path: "subgroup", parent: namespace) + project = create(:project, path: "the-path0", namespace: subgroup) + + subject.rename_namespace(namespace) + + expect(project.route.reload.path).to eq("the-path0/subgroup/the-path0") + end + end + end + + describe '#rename_namespaces' do + context 'top level namespaces' do + let!(:namespace) { create(:namespace, path: 'the-path') } + + it 'should rename the namespace' do + expect(subject).to receive(:rename_namespace). + with(migration_namespace(namespace)) + + subject.rename_namespaces(['the-path'], type: :top_level) + end + end + end +end diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration_spec.rb new file mode 100644 index 00000000000..8b4ab6703c6 --- /dev/null +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration_spec.rb @@ -0,0 +1,29 @@ +require 'spec_helper' + +describe Gitlab::Database::RenameReservedPathsMigration do + let(:subject) do + ActiveRecord::Migration.new.extend( + Gitlab::Database::RenameReservedPathsMigration + ) + end + + describe '#rename_wildcard_paths' do + it 'should rename namespaces' do + expect(subject).to receive(:rename_namespaces). + with(['first-path', 'second-path'], type: :wildcard) + + subject.rename_wildcard_paths(['first-path', 'second-path']) + end + + it 'should rename projects' + end + + describe '#rename_root_paths' do + it 'should rename namespaces' do + expect(subject).to receive(:rename_namespaces). + with(['the-path'], type: :top_level) + + subject.rename_root_paths('the-path') + end + end +end |