Refactor repository paths handling to allow multiple git mount points

7 files changed, 191 insertions, 172 deletions

diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 12d6ac45fb6..e9a4e37ec48 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -356,97 +356,108 @@ namespace :gitlab do
     ########################
 
     def check_repo_base_exists
-      print "Repo base directory exists? ... "
+      puts "Repo base directory exists?"
 
-      repo_base_path = Gitlab.config.gitlab_shell.repos_path
+      Gitlab.config.repositories.storages.each do |name, repo_base_path|
+        print "#{name}... "
 
-      if File.exists?(repo_base_path)
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        puts "#{repo_base_path} is missing".color(:red)
-        try_fixing_it(
-          "This should have been created when setting up GitLab Shell.",
-          "Make sure it's set correctly in config/gitlab.yml",
-          "Make sure GitLab Shell is installed correctly."
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab Shell"
-        )
-        fix_and_rerun
+        if File.exists?(repo_base_path)
+          puts "yes".color(:green)
+        else
+          puts "no".color(:red)
+          puts "#{repo_base_path} is missing".color(:red)
+          try_fixing_it(
+            "This should have been created when setting up GitLab Shell.",
+            "Make sure it's set correctly in config/gitlab.yml",
+            "Make sure GitLab Shell is installed correctly."
+          )
+          for_more_information(
+            see_installation_guide_section "GitLab Shell"
+          )
+          fix_and_rerun
+        end
       end
     end
 
     def check_repo_base_is_not_symlink
-      print "Repo base directory is a symlink? ... "
+      puts "Repo storage directories are symlinks?"
 
-      repo_base_path = Gitlab.config.gitlab_shell.repos_path
-      unless File.exists?(repo_base_path)
-        puts "can't check because of previous errors".color(:magenta)
-        return
-      end
+      Gitlab.config.repositories.storages.each do |name, repo_base_path|
+        print "#{name}... "
 
-      unless File.symlink?(repo_base_path)
-        puts "no".color(:green)
-      else
-        puts "yes".color(:red)
-        try_fixing_it(
-          "Make sure it's set to the real directory in config/gitlab.yml"
-        )
-        fix_and_rerun
+        unless File.exists?(repo_base_path)
+          puts "can't check because of previous errors".color(:magenta)
+          return
+        end
+
+        unless File.symlink?(repo_base_path)
+          puts "no".color(:green)
+        else
+          puts "yes".color(:red)
+          try_fixing_it(
+            "Make sure it's set to the real directory in config/gitlab.yml"
+          )
+          fix_and_rerun
+        end
       end
     end
 
     def check_repo_base_permissions
-      print "Repo base access is drwxrws---? ... "
+      puts "Repo paths access is drwxrws---?"
 
-      repo_base_path = Gitlab.config.gitlab_shell.repos_path
-      unless File.exists?(repo_base_path)
-        puts "can't check because of previous errors".color(:magenta)
-        return
-      end
+      Gitlab.config.repositories.storages.each do |name, repo_base_path|
+        print "#{name}... "
 
-      if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770")
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}",
-          "sudo chmod -R ug-s #{repo_base_path}",
-          "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab Shell"
-        )
-        fix_and_rerun
+        unless File.exists?(repo_base_path)
+          puts "can't check because of previous errors".color(:magenta)
+          return
+        end
+
+        if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770")
+          puts "yes".color(:green)
+        else
+          puts "no".color(:red)
+          try_fixing_it(
+            "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}",
+            "sudo chmod -R ug-s #{repo_base_path}",
+            "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
+          )
+          for_more_information(
+            see_installation_guide_section "GitLab Shell"
+          )
+          fix_and_rerun
+        end
       end
     end
 
     def check_repo_base_user_and_group
       gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
       gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group
-      print "Repo base owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}? ... "
+      puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?"
 
-      repo_base_path = Gitlab.config.gitlab_shell.repos_path
-      unless File.exists?(repo_base_path)
-        puts "can't check because of previous errors".color(:magenta)
-        return
-      end
+      Gitlab.config.repositories.storages.each do |name, repo_base_path|
+        print "#{name}... "
 
-      uid = uid_for(gitlab_shell_ssh_user)
-      gid = gid_for(gitlab_shell_owner_group)
-      if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        puts "  User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue)
-        try_fixing_it(
-          "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab Shell"
-        )
-        fix_and_rerun
+        unless File.exists?(repo_base_path)
+          puts "can't check because of previous errors".color(:magenta)
+          return
+        end
+
+        uid = uid_for(gitlab_shell_ssh_user)
+        gid = gid_for(gitlab_shell_owner_group)
+        if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid
+          puts "yes".color(:green)
+        else
+          puts "no".color(:red)
+          puts "  User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue)
+          try_fixing_it(
+            "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}"
+          )
+          for_more_information(
+            see_installation_guide_section "GitLab Shell"
+          )
+          fix_and_rerun
+        end
       end
     end
 
@@ -473,7 +484,7 @@ namespace :gitlab do
         else
           puts "wrong or missing hooks".color(:red)
           try_fixing_it(
-            sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"),
+            sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"),
             'Check the hooks_path in config/gitlab.yml',
             'Check your gitlab-shell installation'
           )
@@ -785,13 +796,13 @@ namespace :gitlab do
   namespace :repo do
     desc "GitLab | Check the integrity of the repositories managed by GitLab"
     task check: :environment do
-      namespace_dirs = Dir.glob(
-        File.join(Gitlab.config.gitlab_shell.repos_path, '*')
-      )
+      Gitlab.config.repositories.storages.each do |name, path|
+        namespace_dirs = Dir.glob(File.join(path, '*'))
 
-      namespace_dirs.each do |namespace_dir|
-        repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
-        repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
+        namespace_dirs.each do |namespace_dir|
+          repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
+          repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
+        end
       end
     end
   end
@@ -799,12 +810,12 @@ namespace :gitlab do
   namespace :user do
     desc "GitLab | Check the integrity of a specific user's repositories"
     task :check_repos, [:username] => :environment do |t, args|
-      username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue))
+      username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue))
       user = User.find_by(username: username)
       if user
         repo_dirs = user.authorized_projects.map do |p|
                       File.join(
-                        Gitlab.config.gitlab_shell.repos_path,
+                        p.repository_storage_path,
                         "#{p.path_with_namespace}.git"
                       )
                     end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index ab0028d6603..b7cbdc6cd78 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -5,36 +5,36 @@ namespace :gitlab do
       warn_user_is_not_gitlab
       remove_flag = ENV['REMOVE']
 
-
       namespaces = Namespace.pluck(:path)
-      git_base_path = Gitlab.config.gitlab_shell.repos_path
-      all_dirs = Dir.glob(git_base_path + '/*')
+      Gitlab.config.repositories.storages.each do |name, git_base_path|
+        all_dirs = Dir.glob(git_base_path + '/*')
 
-      puts git_base_path.color(:yellow)
-      puts "Looking for directories to remove... "
+        puts git_base_path.color(:yellow)
+        puts "Looking for directories to remove... "
 
-      all_dirs.reject! do |dir|
-        # skip if git repo
-        dir =~ /.git$/
-      end
+        all_dirs.reject! do |dir|
+          # skip if git repo
+          dir =~ /.git$/
+        end
 
-      all_dirs.reject! do |dir|
-        dir_name = File.basename dir
+        all_dirs.reject! do |dir|
+          dir_name = File.basename dir
 
-        # skip if namespace present
-        namespaces.include?(dir_name)
-      end
+          # skip if namespace present
+          namespaces.include?(dir_name)
+        end
 
-      all_dirs.each do |dir_path|
+        all_dirs.each do |dir_path|
 
-        if remove_flag
-          if FileUtils.rm_rf dir_path
-            puts "Removed...#{dir_path}".color(:red)
+          if remove_flag
+            if FileUtils.rm_rf dir_path
+              puts "Removed...#{dir_path}".color(:red)
+            else
+              puts "Cannot remove #{dir_path}".color(:red)
+            end
           else
-            puts "Cannot remove #{dir_path}".color(:red)
+            puts "Can be removed: #{dir_path}".color(:red)
           end
-        else
-          puts "Can be removed: #{dir_path}".color(:red)
         end
       end
 
@@ -48,20 +48,21 @@ namespace :gitlab do
       warn_user_is_not_gitlab
 
       move_suffix = "+orphaned+#{Time.now.to_i}"
-      repo_root = Gitlab.config.gitlab_shell.repos_path
-      # Look for global repos (legacy, depth 1) and normal repos (depth 2)
-      IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
-        find.each_line do |path|
-          path.chomp!
-          repo_with_namespace = path.
-            sub(repo_root, '').
-            sub(%r{^/*}, '').
-            chomp('.git').
-            chomp('.wiki')
-          next if Project.find_with_namespace(repo_with_namespace)
-          new_path = path + move_suffix
-          puts path.inspect + ' -> ' + new_path.inspect
-          File.rename(path, new_path)
+      Gitlab.config.repositories.storages.each do |name, repo_root|
+        # Look for global repos (legacy, depth 1) and normal repos (depth 2)
+        IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
+          find.each_line do |path|
+            path.chomp!
+            repo_with_namespace = path.
+              sub(repo_root, '').
+              sub(%r{^/*}, '').
+              chomp('.git').
+              chomp('.wiki')
+            next if Project.find_with_namespace(repo_with_namespace)
+            new_path = path + move_suffix
+            puts path.inspect + ' -> ' + new_path.inspect
+            File.rename(path, new_path)
+          end
         end
       end
     end
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
index 4753f00c26a..dbdd4e977e8 100644
--- a/lib/tasks/gitlab/import.rake
+++ b/lib/tasks/gitlab/import.rake
@@ -2,73 +2,73 @@ namespace :gitlab do
   namespace :import do
     # How to use:
     #
-    #  1. copy the bare repos under the repos_path (commonly /home/git/repositories)
+    #  1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories)
     #  2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production
     #
     # Notes:
     #  * The project owner will set to the first administator of the system
     #  * Existing projects will be skipped
     #
-    desc "GitLab | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance"
+    desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
     task repos: :environment do
+      Gitlab.config.repositories.storages.each do |name, git_base_path|
+        repos_to_import = Dir.glob(git_base_path + '/**/*.git')
 
-      git_base_path = Gitlab.config.gitlab_shell.repos_path
-      repos_to_import = Dir.glob(git_base_path + '/**/*.git')
+        repos_to_import.each do |repo_path|
+          # strip repo base path
+          repo_path[0..git_base_path.length] = ''
 
-      repos_to_import.each do |repo_path|
-        # strip repo base path
-        repo_path[0..git_base_path.length] = ''
+          path = repo_path.sub(/\.git$/, '')
+          group_name, name = File.split(path)
+          group_name = nil if group_name == '.'
 
-        path = repo_path.sub(/\.git$/, '')
-        group_name, name = File.split(path)
-        group_name = nil if group_name == '.'
+          puts "Processing #{repo_path}".color(:yellow)
 
-        puts "Processing #{repo_path}".color(:yellow)
-
-        if path.end_with?('.wiki')
-          puts " * Skipping wiki repo"
-          next
-        end
+          if path.end_with?('.wiki')
+            puts " * Skipping wiki repo"
+            next
+          end
 
-        project = Project.find_with_namespace(path)
+          project = Project.find_with_namespace(path)
 
-        if project
-          puts " * #{project.name} (#{repo_path}) exists"
-        else
-          user = User.admins.reorder("id").first
+          if project
+            puts " * #{project.name} (#{repo_path}) exists"
+          else
+            user = User.admins.reorder("id").first
 
-          project_params = {
-            name: name,
-            path: name
-          }
+            project_params = {
+              name: name,
+              path: name
+            }
 
-          # find group namespace
-          if group_name
-            group = Namespace.find_by(path: group_name)
-            # create group namespace
-            unless group
-              group = Group.new(:name => group_name)
-              group.path = group_name
-              group.owner = user
-              if group.save
-                puts " * Created Group #{group.name} (#{group.id})".color(:green)
-              else
-                puts " * Failed trying to create group #{group.name}".color(:red)
+            # find group namespace
+            if group_name
+              group = Namespace.find_by(path: group_name)
+              # create group namespace
+              unless group
+                group = Group.new(:name => group_name)
+                group.path = group_name
+                group.owner = user
+                if group.save
+                  puts " * Created Group #{group.name} (#{group.id})".color(:green)
+                else
+                  puts " * Failed trying to create group #{group.name}".color(:red)
+                end
               end
+              # set project group
+              project_params[:namespace_id] = group.id
             end
-            # set project group
-            project_params[:namespace_id] = group.id
-          end
 
-          project = Projects::CreateService.new(user, project_params).execute
+            project = Projects::CreateService.new(user, project_params).execute
 
-          if project.persisted?
-            puts " * Created #{project.name} (#{repo_path})".color(:green)
-            project.update_repository_size
-            project.update_commit_count
-          else
-            puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
-            puts "   Errors: #{project.errors.messages}".color(:red)
+            if project.persisted?
+              puts " * Created #{project.name} (#{repo_path})".color(:green)
+              project.update_repository_size
+              project.update_commit_count
+            else
+              puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
+              puts "   Errors: #{project.errors.messages}".color(:red)
+            end
           end
         end
       end
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index 352b566df24..fe43d40e6d2 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -62,7 +62,10 @@ namespace :gitlab do
       puts ""
       puts "GitLab Shell".color(:yellow)
       puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}"
-      puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}"
+      puts "Repository storage paths:"
+      Gitlab.config.repositories.storages.each do |name, path|
+        puts "- #{name}: \t#{path}"
+      end
       puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}"
       puts "Git:\t\t#{Gitlab.config.git.bin_path}"
 
diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake
index c7596e7abcb..ffcc76e5498 100644
--- a/lib/tasks/gitlab/list_repos.rake
+++ b/lib/tasks/gitlab/list_repos.rake
@@ -9,7 +9,7 @@ namespace :gitlab do
       scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids)
     end
     scope.find_each do |project|
-      base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace)
+      base = File.join(project.repository_storage_path, project.path_with_namespace)
       puts base + '.git'
       puts base + '.wiki.git'
     end
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index b1648a4602a..263798e9c22 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -12,7 +12,6 @@ namespace :gitlab do
       gitlab_url = Gitlab.config.gitlab.url
       # gitlab-shell requires a / at the end of the url
       gitlab_url += '/' unless gitlab_url.end_with?('/')
-      repos_path = Gitlab.config.gitlab_shell.repos_path
       target_dir = Gitlab.config.gitlab_shell.path
 
       # Clone if needed
@@ -35,7 +34,6 @@ namespace :gitlab do
           user: user,
           gitlab_url: gitlab_url,
           http_settings: {self_signed_cert: false}.stringify_keys,
-          repos_path: repos_path,
           auth_file: File.join(home_dir, ".ssh", "authorized_keys"),
           redis: {
             bin: %x{which redis-cli}.chomp,
@@ -58,10 +56,10 @@ namespace :gitlab do
         File.open("config.yml", "w+") {|f| f.puts config.to_yaml}
 
         # Launch installation process
-        system(*%W(bin/install))
+        system(*%W(bin/install) + repository_storage_paths_args)
 
         # (Re)create hooks
-        system(*%W(bin/create-hooks))
+        system(*%W(bin/create-hooks) + repository_storage_paths_args)
       end
 
       # Required for debian packaging with PKGR: Setup .ssh/environment with
@@ -87,7 +85,8 @@ namespace :gitlab do
         if File.exists?(path_to_repo)
           print '-'
         else
-          if Gitlab::Shell.new.add_repository(project.path_with_namespace)
+          if Gitlab::Shell.new.add_repository(project.repository_storage_path,
+                                              project.path_with_namespace)
             print '.'
           else
             print 'F'
@@ -138,4 +137,3 @@ namespace :gitlab do
     system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag}))
   end
 end
-
diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake
index d0c019044b7..ab96b1d3593 100644
--- a/lib/tasks/gitlab/task_helpers.rake
+++ b/lib/tasks/gitlab/task_helpers.rake
@@ -125,10 +125,16 @@ namespace :gitlab do
   end
 
   def all_repos
-    IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
-      find.each_line do |path|
-        yield path.chomp
+    Gitlab.config.repositories.storages.each do |name, path|
+      IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
+        find.each_line do |path|
+          yield path.chomp
+        end
       end
     end
   end
+
+  def repository_storage_paths_args
+    Gitlab.config.repositories.storages.values
+  end
 end