Merge branch 'bvl-rollback-renamed-system-namespace' into 'master'

Don't rename system when migrating from 9.x -> 9.4

Closes #35525 and #36148

See merge request !13228

8 files changed, 45 insertions, 237 deletions

diff --git a/db/migrate/20170316163800_rename_system_namespaces.rb b/db/migrate/20170316163800_rename_system_namespaces.rb
deleted file mode 100644
index 9e9fb5ac225..00000000000
--- a/db/migrate/20170316163800_rename_system_namespaces.rb
+++ /dev/null
@@ -1,231 +0,0 @@
-# See http://doc.gitlab.com/ce/development/migration_style_guide.html
-# for more information on how to write migrations for GitLab.
-class RenameSystemNamespaces < ActiveRecord::Migration
-  include Gitlab::Database::MigrationHelpers
-  include Gitlab::ShellAdapter
-  disable_ddl_transaction!
-
-  class User < ActiveRecord::Base
-    self.table_name = 'users'
-  end
-
-  class Namespace < ActiveRecord::Base
-    self.table_name = 'namespaces'
-    belongs_to :parent, class_name: 'RenameSystemNamespaces::Namespace'
-    has_one :route, as: :source
-    has_many :children, class_name: 'RenameSystemNamespaces::Namespace', foreign_key: :parent_id
-    belongs_to :owner, class_name: 'RenameSystemNamespaces::User'
-
-    # Overridden to have the correct `source_type` for the `route` relation
-    def self.name
-      'Namespace'
-    end
-
-    def full_path
-      if route && route.path.present?
-        @full_path ||= route.path
-      else
-        update_route if persisted?
-
-        build_full_path
-      end
-    end
-
-    def build_full_path
-      if parent && path
-        parent.full_path + '/' + path
-      else
-        path
-      end
-    end
-
-    def update_route
-      prepare_route
-      route.save
-    end
-
-    def prepare_route
-      route || build_route(source: self)
-      route.path = build_full_path
-      route.name = build_full_name
-      @full_path = nil
-      @full_name = nil
-    end
-
-    def build_full_name
-      if parent && name
-        parent.human_name + ' / ' + name
-      else
-        name
-      end
-    end
-
-    def human_name
-      owner&.name
-    end
-  end
-
-  class Route < ActiveRecord::Base
-    self.table_name = 'routes'
-    belongs_to :source, polymorphic: true
-  end
-
-  class Project < ActiveRecord::Base
-    self.table_name = 'projects'
-
-    def repository_storage_path
-      Gitlab.config.repositories.storages[repository_storage]['path']
-    end
-  end
-
-  DOWNTIME = false
-
-  def up
-    return unless system_namespace
-
-    old_path = system_namespace.path
-    old_full_path = system_namespace.full_path
-    # Only remove the last occurrence of the path name to get the parent namespace path
-    namespace_path = remove_last_occurrence(old_full_path, old_path)
-    new_path = rename_path(namespace_path, old_path)
-    new_full_path = join_namespace_path(namespace_path, new_path)
-
-    Namespace.where(id: system_namespace).update_all(path: new_path) # skips callbacks & validations
-
-    replace_statement = replace_sql(Route.arel_table[:path], old_full_path, new_full_path)
-    route_matches = [old_full_path, "#{old_full_path}/%"]
-
-    update_column_in_batches(:routes, :path, replace_statement) do |table, query|
-      query.where(Route.arel_table[:path].matches_any(route_matches))
-    end
-
-    clear_cache_for_namespace(system_namespace)
-
-    # tasks here are based on `Namespace#move_dir`
-    move_repositories(system_namespace, old_full_path, new_full_path)
-    move_namespace_folders(uploads_dir, old_full_path, new_full_path) if file_storage?
-    move_namespace_folders(pages_dir, old_full_path, new_full_path)
-  end
-
-  def down
-    # nothing to do
-  end
-
-  def remove_last_occurrence(string, pattern)
-    string.reverse.sub(pattern.reverse, "").reverse
-  end
-
-  def move_namespace_folders(directory, old_relative_path, new_relative_path)
-    old_path = File.join(directory, old_relative_path)
-    return unless File.directory?(old_path)
-
-    new_path = File.join(directory, new_relative_path)
-    FileUtils.mv(old_path, new_path)
-  end
-
-  def move_repositories(namespace, old_full_path, new_full_path)
-    repo_paths_for_namespace(namespace).each do |repository_storage_path|
-      # Ensure old directory exists before moving it
-      gitlab_shell.add_namespace(repository_storage_path, old_full_path)
-
-      unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
-        say "Exception moving path #{repository_storage_path} from #{old_full_path} to #{new_full_path}"
-      end
-    end
-  end
-
-  def rename_path(namespace_path, path_was)
-    counter = 0
-    path = "#{path_was}#{counter}"
-
-    while route_exists?(join_namespace_path(namespace_path, path))
-      counter += 1
-      path = "#{path_was}#{counter}"
-    end
-
-    path
-  end
-
-  def route_exists?(full_path)
-    Route.where(Route.arel_table[:path].matches(full_path)).any?
-  end
-
-  def join_namespace_path(namespace_path, path)
-    if namespace_path.present?
-      File.join(namespace_path, path)
-    else
-      path
-    end
-  end
-
-  def system_namespace
-    @system_namespace ||= Namespace.where(parent_id: nil)
-                            .where(arel_table[:path].matches(system_namespace_path))
-                            .first
-  end
-
-  def system_namespace_path
-    "system"
-  end
-
-  def clear_cache_for_namespace(namespace)
-    project_ids = projects_for_namespace(namespace).pluck(:id)
-
-    update_column_in_batches(:projects, :description_html, nil) do |table, query|
-      query.where(table[:id].in(project_ids))
-    end
-
-    update_column_in_batches(:issues, :description_html, nil) do |table, query|
-      query.where(table[:project_id].in(project_ids))
-    end
-
-    update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
-      query.where(table[:target_project_id].in(project_ids))
-    end
-
-    update_column_in_batches(:notes, :note_html, nil) do |table, query|
-      query.where(table[:project_id].in(project_ids))
-    end
-
-    update_column_in_batches(:milestones, :description_html, nil) do |table, query|
-      query.where(table[:project_id].in(project_ids))
-    end
-  end
-
-  def projects_for_namespace(namespace)
-    namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id])
-    namespace_or_children = Project.arel_table[:namespace_id].in(namespace_ids)
-    Project.unscoped.where(namespace_or_children)
-  end
-
-  # This won't scale to huge trees, but it should do for a handful of namespaces
-  # called `system`.
-  def child_ids_for_parent(namespace, ids: [])
-    namespace.children.each do |child|
-      ids << child.id
-      child_ids_for_parent(child, ids: ids) if child.children.any?
-    end
-    ids
-  end
-
-  def repo_paths_for_namespace(namespace)
-    projects_for_namespace(namespace).distinct
-      .select(:repository_storage).map(&:repository_storage_path)
-  end
-
-  def uploads_dir
-    File.join(Rails.root, "public", "uploads")
-  end
-
-  def pages_dir
-    Settings.pages.path
-  end
-
-  def file_storage?
-    CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
-  end
-
-  def arel_table
-    Namespace.arel_table
-  end
-end
diff --git a/db/migrate/20170316163845_move_uploads_to_system_dir.rb b/db/migrate/20170316163845_move_uploads_to_system_dir.rb
index 564ee10b5ab..cfcb909ddaf 100644
--- a/db/migrate/20170316163845_move_uploads_to_system_dir.rb
+++ b/db/migrate/20170316163845_move_uploads_to_system_dir.rb
@@ -54,6 +54,6 @@ class MoveUploadsToSystemDir < ActiveRecord::Migration
   end
 
   def new_upload_dir
-    File.join(base_directory, "public", "uploads", "system")
+    File.join(base_directory, "public", "uploads", "-", "system")
   end
 end
diff --git a/db/migrate/20170717074009_move_system_upload_folder.rb b/db/migrate/20170717074009_move_system_upload_folder.rb
index cce31794115..d3caa53a7a4 100644
--- a/db/migrate/20170717074009_move_system_upload_folder.rb
+++ b/db/migrate/20170717074009_move_system_upload_folder.rb
@@ -15,6 +15,11 @@ class MoveSystemUploadFolder < ActiveRecord::Migration
       return
     end
 
+    if File.directory?(new_directory)
+      say "#{new_directory} already exists. No need to redo the move."
+      return
+    end
+
     FileUtils.mkdir_p(File.join(base_directory, '-'))
 
     say "Moving #{old_directory} -> #{new_directory}"
@@ -33,6 +38,11 @@ class MoveSystemUploadFolder < ActiveRecord::Migration
       return
     end
 
+    if !File.symlink?(old_directory) && File.directory?(old_directory)
+      say "#{old_directory} already exists and is not a symlink, no need to revert."
+      return
+    end
+
     if File.symlink?(old_directory)
       say "Removing #{old_directory} -> #{new_directory} symlink"
       FileUtils.rm(old_directory)
diff --git a/db/post_migrate/20170317162059_update_upload_paths_to_system.rb b/db/post_migrate/20170317162059_update_upload_paths_to_system.rb
index ca2912f8dce..92e33848bf0 100644
--- a/db/post_migrate/20170317162059_update_upload_paths_to_system.rb
+++ b/db/post_migrate/20170317162059_update_upload_paths_to_system.rb
@@ -48,7 +48,7 @@ class UpdateUploadPathsToSystem < ActiveRecord::Migration
   end
 
   def new_upload_dir
-    File.join(base_directory, "system")
+    File.join(base_directory, "-",  "system")
   end
 
   def arel_table
diff --git a/db/post_migrate/20170406111121_clean_upload_symlinks.rb b/db/post_migrate/20170406111121_clean_upload_symlinks.rb
index fc3a4acc0bb..f2ce25d4524 100644
--- a/db/post_migrate/20170406111121_clean_upload_symlinks.rb
+++ b/db/post_migrate/20170406111121_clean_upload_symlinks.rb
@@ -47,6 +47,6 @@ class CleanUploadSymlinks < ActiveRecord::Migration
   end
 
   def new_upload_dir
-    File.join(base_directory, "public", "uploads", "system")
+    File.join(base_directory, "public", "uploads", "-", "system")
   end
 end
diff --git a/db/post_migrate/20170606202615_move_appearance_to_system_dir.rb b/db/post_migrate/20170606202615_move_appearance_to_system_dir.rb
index 561de59ec69..07935ab8a52 100644
--- a/db/post_migrate/20170606202615_move_appearance_to_system_dir.rb
+++ b/db/post_migrate/20170606202615_move_appearance_to_system_dir.rb
@@ -52,6 +52,6 @@ class MoveAppearanceToSystemDir < ActiveRecord::Migration
   end
 
   def new_upload_dir
-    File.join(base_directory, "public", "uploads", "system")
+    File.join(base_directory, "public", "uploads", "-", "system")
   end
 end
diff --git a/db/post_migrate/20170612071012_move_personal_snippets_files.rb b/db/post_migrate/20170612071012_move_personal_snippets_files.rb
index 33043364bde..2b79a87ccd8 100644
--- a/db/post_migrate/20170612071012_move_personal_snippets_files.rb
+++ b/db/post_migrate/20170612071012_move_personal_snippets_files.rb
@@ -10,7 +10,7 @@ class MovePersonalSnippetsFiles < ActiveRecord::Migration
     return unless file_storage?
 
     @source_relative_location = File.join('/uploads', 'personal_snippet')
-    @destination_relative_location = File.join('/uploads', 'system', 'personal_snippet')
+    @destination_relative_location = File.join('/uploads', '-', 'system', 'personal_snippet')
 
     move_personal_snippet_files
   end
@@ -18,7 +18,7 @@ class MovePersonalSnippetsFiles < ActiveRecord::Migration
   def down
     return unless file_storage?
 
-    @source_relative_location = File.join('/uploads', 'system', 'personal_snippet')
+    @source_relative_location = File.join('/uploads', '-', 'system', 'personal_snippet')
     @destination_relative_location = File.join('/uploads', 'personal_snippet')
 
     move_personal_snippet_files
diff --git a/db/post_migrate/20170807190736_move_personal_snippet_files_into_correct_folder.rb b/db/post_migrate/20170807190736_move_personal_snippet_files_into_correct_folder.rb
new file mode 100644
index 00000000000..e3d2446b897
--- /dev/null
+++ b/db/post_migrate/20170807190736_move_personal_snippet_files_into_correct_folder.rb
@@ -0,0 +1,29 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class MovePersonalSnippetFilesIntoCorrectFolder < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+  NEW_DIRECTORY = File.join('/uploads', '-', 'system', 'personal_snippet')
+  OLD_DIRECTORY = File.join('/uploads', 'system', 'personal_snippet')
+
+  def up
+    return unless file_storage?
+
+    BackgroundMigrationWorker.perform_async('MovePersonalSnippetFiles',
+                                            [OLD_DIRECTORY, NEW_DIRECTORY])
+  end
+
+  def down
+    return unless file_storage?
+
+    BackgroundMigrationWorker.perform_async('MovePersonalSnippetFiles',
+                                            [NEW_DIRECTORY, OLD_DIRECTORY])
+  end
+
+  def file_storage?
+    CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
+  end
+end