From d981f9aa8a05e72264fcb1eec4119ac75429a3a0 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 25 May 2017 14:43:51 +0200 Subject: Rename all forbidden paths again --- ...170525140254_rename_all_reserved_paths_again.rb | 108 +++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb diff --git a/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb b/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb new file mode 100644 index 00000000000..47c6acc2bff --- /dev/null +++ b/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb @@ -0,0 +1,108 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class RenameAllReservedPathsAgain < ActiveRecord::Migration + include Gitlab::Database::RenameReservedPathsMigration::V1 + + DOWNTIME = false + + disable_ddl_transaction! + + TOP_LEVEL_ROUTES = %w[ + - + .well-known + abuse_reports + admin + all + api + assets + autocomplete + ci + dashboard + explore + files + groups + health_check + help + hooks + import + invites + issues + jwt + koding + member + merge_requests + new + notes + notification_settings + oauth + profile + projects + public + repository + robots.txt + s + search + sent_notifications + services + snippets + teams + u + unicorn_test + unsubscribes + uploads + users + ].freeze + + PROJECT_WILDCARD_ROUTES = %w[ + badges + blame + blob + builds + commits + create + create_dir + edit + environments/folders + files + find_file + gitlab-lfs/objects + info/lfs/objects + new + preview + raw + refs + tree + update + wikis + ].freeze + + GROUP_ROUTES = %w[ + activity + analytics + audit_events + avatar + edit + group_members + hooks + issues + labels + ldap + ldap_group_links + merge_requests + milestones + notification_setting + pipeline_quota + projects + subgroups + ].freeze + + def up + rename_root_paths(TOP_LEVEL_ROUTES) + rename_wildcard_paths(PROJECT_WILDCARD_ROUTES) + rename_child_paths(GROUP_ROUTES) + end + + def down + end +end -- cgit v1.2.1 From 36ecbb6934c73dc7a446c51766b8d43d98be7c12 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Tue, 30 May 2017 12:04:03 +0200 Subject: Rename each route individually --- db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb b/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb index 47c6acc2bff..fd0b68a4154 100644 --- a/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb +++ b/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb @@ -98,9 +98,9 @@ class RenameAllReservedPathsAgain < ActiveRecord::Migration ].freeze def up - rename_root_paths(TOP_LEVEL_ROUTES) - rename_wildcard_paths(PROJECT_WILDCARD_ROUTES) - rename_child_paths(GROUP_ROUTES) + TOP_LEVEL_ROUTES.each { |route| rename_root_paths(route) } + PROJECT_WILDCARD_ROUTES.each { |route| rename_wildcard_paths(route) } + GROUP_ROUTES.each { |route| rename_child_paths(route) } end def down -- cgit v1.2.1 From 3e84b6336f3d61ff56e3e8ef5cc5d8ca08ef0432 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Tue, 6 Jun 2017 12:28:28 +0200 Subject: Track all renames in redis --- .../rename_reserved_paths_migration/v1/rename_base.rb | 5 +++++ .../v1/rename_namespaces.rb | 2 ++ .../v1/rename_projects.rb | 2 ++ .../v1/rename_base_spec.rb | 17 +++++++++++++++++ .../v1/rename_namespaces_spec.rb | 7 +++++++ .../v1/rename_projects_spec.rb | 7 +++++++ spec/support/fake_migration_classes.rb | 4 ++++ 7 files changed, 44 insertions(+) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index d8163d7da11..a8febf4ec62 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -114,6 +114,11 @@ module Gitlab end end + def track_rename(type, old_path, new_path) + key = "rename:#{migration.version}:#{type}" + Gitlab::Redis.with { |redis| redis.lpush(key, [old_path, new_path].to_json) } + end + def file_storage? CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index da7e2cb2e85..a84af02e67b 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -26,6 +26,8 @@ module Gitlab def rename_namespace(namespace) old_full_path, new_full_path = rename_path_for_routable(namespace) + track_rename('namespace', old_full_path, new_full_path) + move_repositories(namespace, old_full_path, new_full_path) move_uploads(old_full_path, new_full_path) move_pages(old_full_path, new_full_path) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 448717eb744..59e47120159 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -16,6 +16,8 @@ module Gitlab def rename_project(project) old_full_path, new_full_path = rename_path_for_routable(project) + track_rename('project', old_full_path, new_full_path) + move_repository(project, old_full_path, new_full_path) move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") move_uploads(old_full_path, new_full_path) diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb index 5653cfee686..fa3c86bbb4f 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb @@ -203,4 +203,21 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca expect(File.exist?(expected_file)).to be(true) end end + + describe '#track_rename', redis: true do + it 'tracks a rename in redis' do + key = 'rename:20170316163845:namespace' + + subject.track_rename('namespace', 'path/to/namespace', 'path/to/renamed') + + old_path, new_path = [nil, nil] + Gitlab::Redis.with do |redis| + rename_info = redis.lpop(key) + old_path, new_path = JSON.parse(rename_info) + end + + expect(old_path).to eq('path/to/namespace') + expect(new_path).to eq('path/to/renamed') + end + end end diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb index 8125dedd3fc..142ebfaf232 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb @@ -191,6 +191,13 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : subject.rename_namespace(user.namespace) end + + it 'tracks the rename' do + expect(subject).to receive(:track_rename) + .with('namespace', 'the-path', 'the-path0') + + subject.rename_namespace(namespace) + end end describe '#rename_user' do diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb index 802f77ad430..ed2f2241ab6 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb @@ -85,6 +85,13 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :tr subject.rename_project(project) end + + it 'tracks the rename' do + expect(subject).to receive(:track_rename) + .with('project', 'known-parent/the-path', 'known-parent/the-path0') + + subject.rename_project(project) + end end describe '#move_repository' do diff --git a/spec/support/fake_migration_classes.rb b/spec/support/fake_migration_classes.rb index 3de0460c3ca..33cd1043132 100644 --- a/spec/support/fake_migration_classes.rb +++ b/spec/support/fake_migration_classes.rb @@ -1,3 +1,7 @@ class FakeRenameReservedPathMigrationV1 < ActiveRecord::Migration include Gitlab::Database::RenameReservedPathsMigration::V1 + + def version + '20170316163845' + end end -- cgit v1.2.1 From 0faff42d7cc62dba50ca99ab1bc034c285311409 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Tue, 6 Jun 2017 17:24:32 +0200 Subject: Add methods to revert project renames --- .../v1/rename_base.rb | 36 ++++++++-- .../v1/rename_projects.rb | 19 ++++++ .../v1/rename_base_spec.rb | 37 +++++++++++ .../v1/rename_projects_spec.rb | 76 +++++++++++++++++----- 4 files changed, 146 insertions(+), 22 deletions(-) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index a8febf4ec62..29f21b08e6c 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -7,6 +7,7 @@ module Gitlab delegate :update_column_in_batches, :replace_sql, + :say, to: :migration def initialize(paths, migration) @@ -26,13 +27,18 @@ module Gitlab new_path = rename_path(namespace_path, old_path) new_full_path = join_routable_path(namespace_path, new_path) + perform_rename(routable, old_full_path, new_full_path) + + [old_full_path, new_full_path] + end + + def perform_rename(routable, old_full_path, new_full_path) # skips callbacks & validations - routable.class.where(id: routable) - .update_all(path: new_path) + new_path = new_full_path.split('/').last + routable.class.where(id: routable). + update_all(path: new_path) rename_routes(old_full_path, new_full_path) - - [old_full_path, new_full_path] end def rename_routes(old_full_path, new_full_path) @@ -86,7 +92,10 @@ module Gitlab def move_folders(directory, old_relative_path, new_relative_path) old_path = File.join(directory, old_relative_path) - return unless File.directory?(old_path) + unless File.directory?(old_path) + say "#{old_path} doesn't exist, skipping" + return + end new_path = File.join(directory, new_relative_path) FileUtils.mv(old_path, new_path) @@ -115,10 +124,25 @@ module Gitlab end def track_rename(type, old_path, new_path) - key = "rename:#{migration.version}:#{type}" + key = redis_key_for_type(type) Gitlab::Redis.with { |redis| redis.lpush(key, [old_path, new_path].to_json) } end + def reverts_for_type(type) + key = redis_key_for_type(type) + Gitlab::Redis.with do |redis| + while rename_info = redis.lpop(key) + path_before_rename, path_after_rename = JSON.parse(rename_info) + say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}" + yield(path_before_rename, path_after_rename) + end + end + end + + def redis_key_for_type(type) + "rename:#{migration.version}:#{type}" + end + def file_storage? CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 59e47120159..1d2a41d91bb 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -18,12 +18,31 @@ module Gitlab track_rename('project', old_full_path, new_full_path) + move_project_folders(project, old_full_path, new_full_path) + end + + def move_project_folders(project, old_full_path, new_full_path) move_repository(project, old_full_path, new_full_path) move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") move_uploads(old_full_path, new_full_path) move_pages(old_full_path, new_full_path) end + def revert_renames + reverts_for_type('project') do |path_before_rename, current_path| + matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) + project = MigrationClasses::Project.joins(:route) + .where(matches_path).first + if project + perform_rename(project, current_path, path_before_rename) + + move_project_folders(project, current_path, path_before_rename) + else + say "Couldn't rename project##{project.id} from #{current_path} back to #{path_before_rename}, project no longer exists" + end + end + end + def move_repository(project, old_path, new_path) unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb index fa3c86bbb4f..4f6f06f0d07 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb @@ -153,6 +153,30 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca end end + describe '#perform_rename' do + describe 'for namespaces' do + let(:namespace) { create(:namespace, path: 'the-path') } + it 'renames the path' do + subject.perform_rename(migration_namespace(namespace), 'the-path', 'renamed') + + expect(namespace.reload.path).to eq('renamed') + end + + it 'renames all the routes for the namespace' do + child = create(:group, path: 'child', parent: namespace) + project = create(:project, namespace: child, path: 'the-project') + other_one = create(:namespace, path: 'the-path-is-similar') + + subject.perform_rename(migration_namespace(namespace), 'the-path', 'renamed') + + expect(namespace.reload.route.path).to eq('renamed') + expect(child.reload.route.path).to eq('renamed/child') + expect(project.reload.route.path).to eq('renamed/child/the-project') + expect(other_one.reload.route.path).to eq('the-path-is-similar') + end + end + end + describe '#move_pages' do it 'moves the pages directory' do expect(subject).to receive(:move_folders) @@ -220,4 +244,17 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca expect(new_path).to eq('path/to/renamed') end end + + describe '#reverts_for_type', redis: true do + it 'yields for each tracked rename' do + subject.track_rename('project', 'old_path', 'new_path') + subject.track_rename('project', 'old_path2', 'new_path2') + subject.track_rename('namespace', 'namespace_path', 'new_namespace_path') + + expect { |b| subject.reverts_for_type('project', &b) } + .to yield_successive_args(%w(old_path2 new_path2), %w(old_path new_path)) + expect { |b| subject.reverts_for_type('namespace', &b) } + .to yield_with_args('namespace_path', 'new_namespace_path') + end + end end diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb index ed2f2241ab6..f87f4371353 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb @@ -3,6 +3,12 @@ require 'spec_helper' describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :truncate do let(:migration) { FakeRenameReservedPathMigrationV1.new } let(:subject) { described_class.new(['the-path'], migration) } + let(:project) do + create(:empty_project, + path: 'the-path', + namespace: create(:namespace, path: 'known-parent' )) + end + before do allow(migration).to receive(:say) @@ -47,12 +53,6 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :tr end describe '#rename_project' do - let(:project) do - create(:empty_project, - path: 'the-path', - namespace: create(:namespace, path: 'known-parent' )) - end - it 'renames path & route for the project' do expect(subject).to receive(:rename_path_for_routable) .with(project) @@ -63,34 +63,42 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :tr expect(project.reload.path).to eq('the-path0') end + it 'tracks the rename' do + expect(subject).to receive(:track_rename) + .with('project', 'known-parent/the-path', 'known-parent/the-path0') + + subject.rename_project(project) + end + + it 'renames the folders for the project' do + expect(subject).to receive(:move_project_folders).with(project, 'known-parent/the-path', 'known-parent/the-path0') + + subject.rename_project(project) + end + end + + describe '#move_project_folders' do it 'moves the wiki & the repo' do expect(subject).to receive(:move_repository) .with(project, 'known-parent/the-path.wiki', 'known-parent/the-path0.wiki') expect(subject).to receive(:move_repository) .with(project, 'known-parent/the-path', 'known-parent/the-path0') - subject.rename_project(project) + subject.move_project_folders(project, 'known-parent/the-path', 'known-parent/the-path0') end it 'moves uploads' do expect(subject).to receive(:move_uploads) .with('known-parent/the-path', 'known-parent/the-path0') - subject.rename_project(project) + subject.move_project_folders(project, 'known-parent/the-path', 'known-parent/the-path0') end it 'moves pages' do expect(subject).to receive(:move_pages) .with('known-parent/the-path', 'known-parent/the-path0') - subject.rename_project(project) - end - - it 'tracks the rename' do - expect(subject).to receive(:track_rename) - .with('project', 'known-parent/the-path', 'known-parent/the-path0') - - subject.rename_project(project) + subject.move_project_folders(project, 'known-parent/the-path', 'known-parent/the-path0') end end @@ -106,4 +114,40 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :tr expect(File.directory?(expected_path)).to be(true) end end + + describe '#revert_renames', redis: true do + it 'renames the routes back to the previous values' do + subject.rename_project(project) + + expect(subject).to receive(:perform_rename) + .with( + kind_of(Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::Project), + 'known-parent/the-path0', + 'known-parent/the-path' + ).and_call_original + + subject.revert_renames + + expect(project.reload.path).to eq('the-path') + expect(project.route.path).to eq('known-parent/the-path') + end + + it 'moves the repositories back to their original place' do + project.create_repository + subject.rename_project(project) + + expected_path = File.join(TestEnv.repos_path, 'known-parent', 'the-path.git') + + expect(subject).to receive(:move_project_folders) + .with( + kind_of(Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::Project), + 'known-parent/the-path0', + 'known-parent/the-path' + ).and_call_original + + subject.revert_renames + + expect(File.directory?(expected_path)).to be_truthy + end + end end -- cgit v1.2.1 From 152cba56e4004de1b1c2accee77e40a019d8c667 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Tue, 6 Jun 2017 18:40:11 +0200 Subject: Revert namespace renames --- .../v1/rename_namespaces.rb | 21 ++++++ .../v1/rename_projects.rb | 3 +- .../v1/rename_base_spec.rb | 1 + .../v1/rename_namespaces_spec.rb | 78 +++++++++++++++++----- .../v1/rename_projects_spec.rb | 2 +- 5 files changed, 88 insertions(+), 17 deletions(-) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index a84af02e67b..434167b0f88 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -28,6 +28,10 @@ module Gitlab track_rename('namespace', old_full_path, new_full_path) + rename_namespace_dependencies(namespace, old_full_path, new_full_path) + end + + def rename_namespace_dependencies(namespace, old_full_path, new_full_path) move_repositories(namespace, old_full_path, new_full_path) move_uploads(old_full_path, new_full_path) move_pages(old_full_path, new_full_path) @@ -35,6 +39,23 @@ module Gitlab remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id)) end + def revert_renames + reverts_for_type('namespace') do |path_before_rename, current_path| + matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) + namespace = MigrationClasses::Namespace.joins(:route) + .where(matches_path).first.becomes(MigrationClasses::Namespace) + + if namespace + perform_rename(namespace, current_path, path_before_rename) + + rename_namespace_dependencies(namespace, current_path, path_before_rename) + else + say "Couldn't rename namespace##{namespace.id} from #{current_path} "\ + " back to #{path_before_rename}, namespace no longer exists" + end + end + end + def rename_user(old_username, new_username) MigrationClasses::User.where(username: old_username) .update_all(username: new_username) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 1d2a41d91bb..061f531f1d5 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -38,7 +38,8 @@ module Gitlab move_project_folders(project, current_path, path_before_rename) else - say "Couldn't rename project##{project.id} from #{current_path} back to #{path_before_rename}, project no longer exists" + say "Couldn't rename project##{project.id} from #{current_path} "\ + "back to #{path_before_rename}, project no longer exists" end end end diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb index 4f6f06f0d07..f4e19f09419 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb @@ -6,6 +6,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca before do allow(migration).to receive(:say) + TestEnv.clean_test_path end def migration_namespace(namespace) diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb index 142ebfaf232..5fa3d809c7a 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb @@ -3,9 +3,11 @@ require 'spec_helper' describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :truncate do let(:migration) { FakeRenameReservedPathMigrationV1.new } let(:subject) { described_class.new(['the-path'], migration) } + let(:namespace) { create(:group, name: 'the-path') } before do allow(migration).to receive(:say) + TestEnv.clean_test_path end def migration_namespace(namespace) @@ -137,8 +139,6 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : end describe "#rename_namespace" do - let(:namespace) { create(:group, name: 'the-path') } - it 'renames paths & routes for the namespace' do expect(subject).to receive(:rename_path_for_routable) .with(namespace) @@ -149,11 +149,27 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : expect(namespace.reload.path).to eq('the-path0') end + it 'tracks the rename' do + expect(subject).to receive(:track_rename) + .with('namespace', 'the-path', 'the-path0') + + subject.rename_namespace(namespace) + end + + it 'renames things related to the namespace' do + expect(subject).to receive(:rename_namespace_dependencies) + .with(namespace, 'the-path', 'the-path0') + + subject.rename_namespace(namespace) + end + end + + describe '#rename_namespace_dependencies' do it "moves the the repository for a project in the namespace" do create(:project, namespace: namespace, path: "the-path-project") expected_repo = File.join(TestEnv.repos_path, "the-path0", "the-path-project.git") - subject.rename_namespace(namespace) + subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0') expect(File.directory?(expected_repo)).to be(true) end @@ -161,13 +177,13 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : it "moves the uploads for the namespace" do expect(subject).to receive(:move_uploads).with("the-path", "the-path0") - subject.rename_namespace(namespace) + subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0') end it "moves the pages for the namespace" do expect(subject).to receive(:move_pages).with("the-path", "the-path0") - subject.rename_namespace(namespace) + subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0') end it 'invalidates the markdown cache of related projects' do @@ -175,13 +191,13 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : expect(subject).to receive(:remove_cached_html_for_projects).with([project.id]) - subject.rename_namespace(namespace) + subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0') end it "doesn't rename users for other namespaces" do expect(subject).not_to receive(:rename_user) - subject.rename_namespace(namespace) + subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0') end it 'renames the username of a namespace for a user' do @@ -189,14 +205,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : expect(subject).to receive(:rename_user).with('the-path', 'the-path0') - subject.rename_namespace(user.namespace) - end - - it 'tracks the rename' do - expect(subject).to receive(:track_rename) - .with('namespace', 'the-path', 'the-path0') - - subject.rename_namespace(namespace) + subject.rename_namespace_dependencies(user.namespace, 'the-path', 'the-path0') end end @@ -231,4 +240,43 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : subject.rename_namespaces(type: :child) end end + + describe '#revert_renames', redis: true do + it 'renames the routes back to the previous values' do + project = create(:project, path: 'a-project', namespace: namespace) + subject.rename_namespace(namespace) + + expect(subject).to receive(:perform_rename) + .with( + kind_of(Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::Namespace), + 'the-path0', + 'the-path' + ).and_call_original + + subject.revert_renames + + expect(namespace.reload.path).to eq('the-path') + expect(namespace.reload.route.path).to eq('the-path') + expect(project.reload.route.path).to eq('the-path/a-project') + end + + it 'moves the repositories back to their original place' do + project = create(:project, path: 'a-project', namespace: namespace) + project.create_repository + subject.rename_namespace(namespace) + + expected_path = File.join(TestEnv.repos_path, 'the-path', 'a-project.git') + + expect(subject).to receive(:rename_namespace_dependencies) + .with( + kind_of(Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::Namespace), + 'the-path0', + 'the-path' + ).and_call_original + + subject.revert_renames + + expect(File.directory?(expected_path)).to be_truthy + end + end end diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb index f87f4371353..ca741a4989b 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb @@ -9,9 +9,9 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :tr namespace: create(:namespace, path: 'known-parent' )) end - before do allow(migration).to receive(:say) + TestEnv.clean_test_path end describe '#projects_for_paths' do -- cgit v1.2.1 From c98ed42d01d4a15ce2a588079601b7d620b029ff Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 12 Jun 2017 11:03:28 +0200 Subject: Revert renames from a migration --- ...170525140254_rename_all_reserved_paths_again.rb | 1 + .../database/rename_reserved_paths_migration/v1.rb | 5 +++++ .../rename_reserved_paths_migration/v1_spec.rb | 22 ++++++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb b/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb index fd0b68a4154..9ec59913b66 100644 --- a/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb +++ b/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb @@ -104,5 +104,6 @@ class RenameAllReservedPathsAgain < ActiveRecord::Migration end def down + revert_renames end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb index 89530082cd2..f333ff22300 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb @@ -29,6 +29,11 @@ module Gitlab paths = Array(paths) RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level) end + + def revert_renames + RenameProjects.new([], self).revert_renames + RenameNamespaces.new([], self).revert_renames + end end end end diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1_spec.rb index 1d5e58855c1..9a48bc3a048 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1_spec.rb @@ -51,4 +51,26 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1, :truncate do subject.rename_root_paths('the-path') end end + + describe '#revert_renames' do + it 'renames namespaces' do + rename_namespaces = double + expect(described_class::RenameNamespaces). + to receive(:new).with([], subject). + and_return(rename_namespaces) + expect(rename_namespaces).to receive(:revert_renames) + + subject.revert_renames + end + + it 'renames projects' do + rename_projects = double + expect(described_class::RenameProjects). + to receive(:new).with([], subject). + and_return(rename_projects) + expect(rename_projects).to receive(:revert_renames) + + subject.revert_renames + end + end end -- cgit v1.2.1 From 1ebb2255362788b8c34fb4120ed9e2ba478ee53b Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 12 Jun 2017 14:16:03 +0200 Subject: More logging so we know we have the rename in redis --- .../database/rename_reserved_paths_migration/v1/rename_base.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 29f21b08e6c..3bf549a56eb 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -125,7 +125,11 @@ module Gitlab def track_rename(type, old_path, new_path) key = redis_key_for_type(type) - Gitlab::Redis.with { |redis| redis.lpush(key, [old_path, new_path].to_json) } + Gitlab::Redis.with do |redis| + redis.lpush(key, [old_path, new_path].to_json) + redis.expire(key, 2.weeks.to_i) + end + say "tracked rename: #{key}: #{old_path} -> #{new_path}" end def reverts_for_type(type) -- cgit v1.2.1 From 229ac39a4c7f7cc4fa207ffa1c826e114df2906a Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 12 Jun 2017 19:10:00 +0200 Subject: Don't break rolling back when a namespace or project was renamed --- .../rename_reserved_paths_migration/v1/rename_namespaces.rb | 6 +++--- .../database/rename_reserved_paths_migration/v1/rename_projects.rb | 6 ++++-- .../rename_reserved_paths_migration/v1/rename_namespaces_spec.rb | 7 +++++++ .../rename_reserved_paths_migration/v1/rename_projects_spec.rb | 7 +++++++ 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 434167b0f88..f85f2d90f02 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -43,15 +43,15 @@ module Gitlab reverts_for_type('namespace') do |path_before_rename, current_path| matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) namespace = MigrationClasses::Namespace.joins(:route) - .where(matches_path).first.becomes(MigrationClasses::Namespace) + .where(matches_path).first&.becomes(MigrationClasses::Namespace) if namespace perform_rename(namespace, current_path, path_before_rename) rename_namespace_dependencies(namespace, current_path, path_before_rename) else - say "Couldn't rename namespace##{namespace.id} from #{current_path} "\ - " back to #{path_before_rename}, namespace no longer exists" + say "Couldn't rename namespace from #{current_path} back to #{path_before_rename} "\ + "namespace was renamed, or no longer exists at the expected path" end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 061f531f1d5..8f00e957f85 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -38,8 +38,10 @@ module Gitlab move_project_folders(project, current_path, path_before_rename) else - say "Couldn't rename project##{project.id} from #{current_path} "\ - "back to #{path_before_rename}, project no longer exists" + say "Couldn't rename Project from #{current_path} back to "\ + "#{path_before_rename}, project was renamed or no longer "\ + "exists at the expected path." + end end end diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb index 5fa3d809c7a..803e923b4a5 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb @@ -278,5 +278,12 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : expect(File.directory?(expected_path)).to be_truthy end + + it "doesn't break when the namespace was renamed" do + subject.rename_namespace(namespace) + namespace.update_attributes!(path: 'renamed-afterwards') + + expect { subject.revert_renames }.not_to raise_error + end end end diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb index ca741a4989b..0e240a5ccf1 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb @@ -149,5 +149,12 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :tr expect(File.directory?(expected_path)).to be_truthy end + + it "doesn't break when the project was renamed" do + subject.rename_project(project) + project.update_attributes!(path: 'renamed-afterwards') + + expect { subject.revert_renames }.not_to raise_error + end end end -- cgit v1.2.1 From d6a0c288c89765fa8f0e96aedefc608dd7025491 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 12 Jun 2017 19:19:00 +0200 Subject: Use the migration name as a key in redis --- lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb | 2 +- .../database/rename_reserved_paths_migration/v1/rename_base_spec.rb | 2 +- spec/support/fake_migration_classes.rb | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 3bf549a56eb..ce82a57fe7e 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -144,7 +144,7 @@ module Gitlab end def redis_key_for_type(type) - "rename:#{migration.version}:#{type}" + "rename:#{migration.name}:#{type}" end def file_storage? diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb index f4e19f09419..2ce3fc0b497 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb @@ -231,7 +231,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca describe '#track_rename', redis: true do it 'tracks a rename in redis' do - key = 'rename:20170316163845:namespace' + key = 'rename:FakeRenameReservedPathMigrationV1:namespace' subject.track_rename('namespace', 'path/to/namespace', 'path/to/renamed') diff --git a/spec/support/fake_migration_classes.rb b/spec/support/fake_migration_classes.rb index 33cd1043132..b0fc8422857 100644 --- a/spec/support/fake_migration_classes.rb +++ b/spec/support/fake_migration_classes.rb @@ -4,4 +4,8 @@ class FakeRenameReservedPathMigrationV1 < ActiveRecord::Migration def version '20170316163845' end + + def name + "FakeRenameReservedPathMigrationV1" + end end -- cgit v1.2.1 From 171f2d97dcc508ff6031248c6f78fc8ba75e939c Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 14 Jun 2017 15:20:03 +0200 Subject: Keep failed renames in redis --- .../rename_reserved_paths_migration/v1/rename_base.rb | 14 +++++++++++++- .../v1/rename_base_spec.rb | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index ce82a57fe7e..060203fda12 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -134,12 +134,24 @@ module Gitlab def reverts_for_type(type) key = redis_key_for_type(type) + Gitlab::Redis.with do |redis| + failed_reverts = [] + while rename_info = redis.lpop(key) path_before_rename, path_after_rename = JSON.parse(rename_info) say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}" - yield(path_before_rename, path_after_rename) + begin + yield(path_before_rename, path_after_rename) + rescue StandardError => e + failed_reverts << rename_info + say "Renaming #{type} from back to #{path_before_rename} failed. "\ + "Review the error and try again by running the `down` action. \n"\ + "#{e.message}: \n #{e.backtrace.join("\n")}" + end end + + failed_reverts.each { |rename_info| redis.lpush(key, rename_info) } end end diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb index 2ce3fc0b497..8813f129ef5 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb @@ -257,5 +257,24 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca expect { |b| subject.reverts_for_type('namespace', &b) } .to yield_with_args('namespace_path', 'new_namespace_path') end + + it 'keeps the revert in redis if it failed' do + subject.track_rename('project', 'old_path', 'new_path') + + subject.reverts_for_type('project') do + raise 'whatever happens, keep going!' + end + + key = 'rename:FakeRenameReservedPathMigrationV1:project' + stored_renames = nil + rename_count = 0 + Gitlab::Redis.with do |redis| + stored_renames = redis.lrange(key, 0, 1) + rename_count = redis.llen(key) + end + + expect(rename_count).to eq(1) + expect(JSON.parse(stored_renames.first)).to eq(%w(old_path new_path)) + end end end -- cgit v1.2.1 From 8c850380a9908542c50c7dc1226bf99274eb48e6 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 14 Jun 2017 15:21:51 +0200 Subject: Add punctuation to log messages --- .../database/rename_reserved_paths_migration/v1/rename_base.rb | 5 +++-- .../database/rename_reserved_paths_migration/v1/rename_namespaces.rb | 2 +- .../database/rename_reserved_paths_migration/v1/rename_projects.rb | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 060203fda12..68dd6f9a5b3 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -145,8 +145,9 @@ module Gitlab yield(path_before_rename, path_after_rename) rescue StandardError => e failed_reverts << rename_info - say "Renaming #{type} from back to #{path_before_rename} failed. "\ - "Review the error and try again by running the `down` action. \n"\ + say "Renaming #{type} from #{path_after_rename} back to "\ + "#{path_before_rename} failed. Review the error and try "\ + "again by running the `down` action. \n"\ "#{e.message}: \n #{e.backtrace.join("\n")}" end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index f85f2d90f02..05b86f32ce2 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -50,7 +50,7 @@ module Gitlab rename_namespace_dependencies(namespace, current_path, path_before_rename) else - say "Couldn't rename namespace from #{current_path} back to #{path_before_rename} "\ + say "Couldn't rename namespace from #{current_path} back to #{path_before_rename}, "\ "namespace was renamed, or no longer exists at the expected path" end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 8f00e957f85..75a75f61953 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -33,12 +33,13 @@ module Gitlab matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) project = MigrationClasses::Project.joins(:route) .where(matches_path).first + if project perform_rename(project, current_path, path_before_rename) move_project_folders(project, current_path, path_before_rename) else - say "Couldn't rename Project from #{current_path} back to "\ + say "Couldn't rename project from #{current_path} back to "\ "#{path_before_rename}, project was renamed or no longer "\ "exists at the expected path." -- cgit v1.2.1 From 589b0da7c84280fc24a3b0f1773d0de8022c291e Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 16 Jun 2017 13:39:23 +0200 Subject: Add changelog --- changelogs/unreleased/bvl-rename-all-reserved-paths.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelogs/unreleased/bvl-rename-all-reserved-paths.yml diff --git a/changelogs/unreleased/bvl-rename-all-reserved-paths.yml b/changelogs/unreleased/bvl-rename-all-reserved-paths.yml new file mode 100644 index 00000000000..f37f2fa94ae --- /dev/null +++ b/changelogs/unreleased/bvl-rename-all-reserved-paths.yml @@ -0,0 +1,4 @@ +--- +title: Rename all reserved paths that could have been created +merge_request: 11713 +author: -- cgit v1.2.1 From 66ba0b0a3444eb5063803df316be1cd14e3ef605 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 21 Jun 2017 15:27:25 +0200 Subject: Clear the cache for projects one-by-one --- .../v1/rename_base.rb | 30 ++++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 68dd6f9a5b3..74f6984a6bb 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -102,24 +102,26 @@ module Gitlab end def remove_cached_html_for_projects(project_ids) - update_column_in_batches(:projects, :description_html, nil) do |table, query| - query.where(table[:id].in(project_ids)) - end + project_ids.each do |project_id| + update_column_in_batches(:projects, :description_html, nil) do |table, query| + query.where(table[:id].eq(project_id)) + end - update_column_in_batches(:issues, :description_html, nil) do |table, query| - query.where(table[:project_id].in(project_ids)) - end + update_column_in_batches(:issues, :description_html, nil) do |table, query| + query.where(table[:project_id].eq(project_id)) + end - update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| - query.where(table[:target_project_id].in(project_ids)) - end + update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| + query.where(table[:target_project_id].eq(project_id)) + end - update_column_in_batches(:notes, :note_html, nil) do |table, query| - query.where(table[:project_id].in(project_ids)) - end + update_column_in_batches(:notes, :note_html, nil) do |table, query| + query.where(table[:project_id].eq(project_id)) + end - update_column_in_batches(:milestones, :description_html, nil) do |table, query| - query.where(table[:project_id].in(project_ids)) + update_column_in_batches(:milestones, :description_html, nil) do |table, query| + query.where(table[:project_id].eq(project_id)) + end end end -- cgit v1.2.1 From 79cdacc5d464cfb47076efe27e3d6ffe32ff1dad Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 21 Jun 2017 15:37:21 +0200 Subject: Disable statement timeout --- db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb b/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb index 9ec59913b66..9441b236c8d 100644 --- a/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb +++ b/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb @@ -98,12 +98,16 @@ class RenameAllReservedPathsAgain < ActiveRecord::Migration ].freeze def up + disable_statement_timeout + TOP_LEVEL_ROUTES.each { |route| rename_root_paths(route) } PROJECT_WILDCARD_ROUTES.each { |route| rename_wildcard_paths(route) } GROUP_ROUTES.each { |route| rename_child_paths(route) } end def down + disable_statement_timeout + revert_renames end end -- cgit v1.2.1 From 60561aca22f7abbace1b46bc8312cd2192c02344 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 23 Jun 2017 20:59:38 +0200 Subject: Update routes directly by ID instead of filtering by path --- .../rename_reserved_paths_migration/v1/rename_base.rb | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 74f6984a6bb..c7ce9749eba 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -6,6 +6,7 @@ module Gitlab attr_reader :paths, :migration delegate :update_column_in_batches, + :execute, :replace_sql, :say, to: :migration @@ -42,14 +43,21 @@ module Gitlab end def rename_routes(old_full_path, new_full_path) + routes = Route.arel_table + main_route_ids = routes.project(routes[:id]).where(routes[:path].matches(old_full_path)) + child_route_ids = routes.project(routes[:id]).where(routes[:path].matches("#{old_full_path}/%")) + matching_ids = main_route_ids.union(child_route_ids) + ids = execute(matching_ids.to_sql).map { |entry| entry['id'] } + replace_statement = replace_sql(Route.arel_table[:path], old_full_path, new_full_path) - update_column_in_batches(:routes, :path, replace_statement) do |table, query| - path_or_children = table[:path].matches_any([old_full_path, "#{old_full_path}/%"]) - query.where(path_or_children) - end + update = Arel::UpdateManager.new(ActiveRecord::Base) + .table(routes) + .set([[routes[:path], replace_statement]]) + .where(routes[:id].in(ids)) + execute(update.to_sql) end def rename_path(namespace_path, path_was) -- cgit v1.2.1 From 397d3fd7d0cd352c94637a26b5ed25c025aa8bf9 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 26 Jun 2017 17:45:32 +0200 Subject: Only do one query for updating routes --- .../v1/rename_base.rb | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index c7ce9749eba..c696c8baf7e 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -8,6 +8,7 @@ module Gitlab delegate :update_column_in_batches, :execute, :replace_sql, + :quote_string, :say, to: :migration @@ -44,10 +45,18 @@ module Gitlab def rename_routes(old_full_path, new_full_path) routes = Route.arel_table - main_route_ids = routes.project(routes[:id]).where(routes[:path].matches(old_full_path)) - child_route_ids = routes.project(routes[:id]).where(routes[:path].matches("#{old_full_path}/%")) - matching_ids = main_route_ids.union(child_route_ids) - ids = execute(matching_ids.to_sql).map { |entry| entry['id'] } + + quoted_old_full_path = quote_string(old_full_path) + quoted_old_wildcard_path = quote_string("#{old_full_path}/%") + + filter = if Database.mysql? + "lower(routes.path) = lower('#{quoted_old_full_path}') "\ + "OR routes.path LIKE '#{quoted_old_wildcard_path}'" + else + "routes.id IN "\ + "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ + "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" + end replace_statement = replace_sql(Route.arel_table[:path], old_full_path, @@ -56,7 +65,8 @@ module Gitlab update = Arel::UpdateManager.new(ActiveRecord::Base) .table(routes) .set([[routes[:path], replace_statement]]) - .where(routes[:id].in(ids)) + .where(Arel::Nodes::SqlLiteral.new(filter)) + execute(update.to_sql) end -- cgit v1.2.1 From 7c53fcf11fff8a05442ae813c9da76a9ed710abd Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Tue, 27 Jun 2017 13:05:02 +0200 Subject: Adjust for new static-analysis failures --- .../rename_reserved_paths_migration/v1/rename_base.rb | 4 ++-- .../database/rename_reserved_paths_migration/v1_spec.rb | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index c696c8baf7e..33f8939bc61 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -37,8 +37,8 @@ module Gitlab def perform_rename(routable, old_full_path, new_full_path) # skips callbacks & validations new_path = new_full_path.split('/').last - routable.class.where(id: routable). - update_all(path: new_path) + routable.class.where(id: routable) + .update_all(path: new_path) rename_routes(old_full_path, new_full_path) end diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1_spec.rb index 9a48bc3a048..7695b95dc57 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1_spec.rb @@ -55,9 +55,9 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1, :truncate do describe '#revert_renames' do it 'renames namespaces' do rename_namespaces = double - expect(described_class::RenameNamespaces). - to receive(:new).with([], subject). - and_return(rename_namespaces) + expect(described_class::RenameNamespaces) + .to receive(:new).with([], subject) + .and_return(rename_namespaces) expect(rename_namespaces).to receive(:revert_renames) subject.revert_renames @@ -65,9 +65,9 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1, :truncate do it 'renames projects' do rename_projects = double - expect(described_class::RenameProjects). - to receive(:new).with([], subject). - and_return(rename_projects) + expect(described_class::RenameProjects) + .to receive(:new).with([], subject) + .and_return(rename_projects) expect(rename_projects).to receive(:revert_renames) subject.revert_renames -- cgit v1.2.1