diff options
author | Alejandro RodrÃguez <alejorro70@gmail.com> | 2016-06-22 17:04:51 -0400 |
---|---|---|
committer | Alejandro RodrÃguez <alejorro70@gmail.com> | 2016-06-29 22:30:31 -0400 |
commit | 86359ec854314574dccea75247f45590262b05c0 (patch) | |
tree | 52aaf922e5a6fc63b0d42095322f79cdae659b43 /lib/tasks | |
parent | b32a6add8fa602eb35648f3f4661df8b98d909cb (diff) | |
download | gitlab-ce-86359ec854314574dccea75247f45590262b05c0.tar.gz |
Refactor repository paths handling to allow multiple git mount points
Diffstat (limited to 'lib/tasks')
-rw-r--r-- | lib/tasks/gitlab/check.rake | 167 | ||||
-rw-r--r-- | lib/tasks/gitlab/cleanup.rake | 71 | ||||
-rw-r--r-- | lib/tasks/gitlab/import.rake | 96 | ||||
-rw-r--r-- | lib/tasks/gitlab/info.rake | 5 | ||||
-rw-r--r-- | lib/tasks/gitlab/list_repos.rake | 2 | ||||
-rw-r--r-- | lib/tasks/gitlab/shell.rake | 10 | ||||
-rw-r--r-- | lib/tasks/gitlab/task_helpers.rake | 12 |
7 files changed, 191 insertions, 172 deletions
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 12d6ac45fb6..e9a4e37ec48 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -356,97 +356,108 @@ namespace :gitlab do ######################## def check_repo_base_exists - print "Repo base directory exists? ... " + puts "Repo base directory exists?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.exists?(repo_base_path) - puts "yes".color(:green) - else - puts "no".color(:red) - puts "#{repo_base_path} is missing".color(:red) - try_fixing_it( - "This should have been created when setting up GitLab Shell.", - "Make sure it's set correctly in config/gitlab.yml", - "Make sure GitLab Shell is installed correctly." - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + if File.exists?(repo_base_path) + puts "yes".color(:green) + else + puts "no".color(:red) + puts "#{repo_base_path} is missing".color(:red) + try_fixing_it( + "This should have been created when setting up GitLab Shell.", + "Make sure it's set correctly in config/gitlab.yml", + "Make sure GitLab Shell is installed correctly." + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_is_not_symlink - print "Repo base directory is a symlink? ... " + puts "Repo storage directories are symlinks?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - unless File.symlink?(repo_base_path) - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Make sure it's set to the real directory in config/gitlab.yml" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + unless File.symlink?(repo_base_path) + puts "no".color(:green) + else + puts "yes".color(:red) + try_fixing_it( + "Make sure it's set to the real directory in config/gitlab.yml" + ) + fix_and_rerun + end end end def check_repo_base_permissions - print "Repo base access is drwxrws---? ... " + puts "Repo paths access is drwxrws---?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", - "sudo chmod -R ug-s #{repo_base_path}", - "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") + puts "yes".color(:green) + else + puts "no".color(:red) + try_fixing_it( + "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", + "sudo chmod -R ug-s #{repo_base_path}", + "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_user_and_group gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group - print "Repo base owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}? ... " + puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - uid = uid_for(gitlab_shell_ssh_user) - gid = gid_for(gitlab_shell_owner_group) - if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid - puts "yes".color(:green) - else - puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) - try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + uid = uid_for(gitlab_shell_ssh_user) + gid = gid_for(gitlab_shell_owner_group) + if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid + puts "yes".color(:green) + else + puts "no".color(:red) + puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) + try_fixing_it( + "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end @@ -473,7 +484,7 @@ namespace :gitlab do else puts "wrong or missing hooks".color(:red) try_fixing_it( - sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"), + sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"), 'Check the hooks_path in config/gitlab.yml', 'Check your gitlab-shell installation' ) @@ -785,13 +796,13 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - namespace_dirs = Dir.glob( - File.join(Gitlab.config.gitlab_shell.repos_path, '*') - ) + Gitlab.config.repositories.storages.each do |name, path| + namespace_dirs = Dir.glob(File.join(path, '*')) - namespace_dirs.each do |namespace_dir| - repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + namespace_dirs.each do |namespace_dir| + repo_dirs = Dir.glob(File.join(namespace_dir, '*')) + repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + end end end end @@ -799,12 +810,12 @@ namespace :gitlab do namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| - username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue)) + username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| File.join( - Gitlab.config.gitlab_shell.repos_path, + p.repository_storage_path, "#{p.path_with_namespace}.git" ) end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index ab0028d6603..b7cbdc6cd78 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -5,36 +5,36 @@ namespace :gitlab do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] - namespaces = Namespace.pluck(:path) - git_base_path = Gitlab.config.gitlab_shell.repos_path - all_dirs = Dir.glob(git_base_path + '/*') + Gitlab.config.repositories.storages.each do |name, git_base_path| + all_dirs = Dir.glob(git_base_path + '/*') - puts git_base_path.color(:yellow) - puts "Looking for directories to remove... " + puts git_base_path.color(:yellow) + puts "Looking for directories to remove... " - all_dirs.reject! do |dir| - # skip if git repo - dir =~ /.git$/ - end + all_dirs.reject! do |dir| + # skip if git repo + dir =~ /.git$/ + end - all_dirs.reject! do |dir| - dir_name = File.basename dir + all_dirs.reject! do |dir| + dir_name = File.basename dir - # skip if namespace present - namespaces.include?(dir_name) - end + # skip if namespace present + namespaces.include?(dir_name) + end - all_dirs.each do |dir_path| + all_dirs.each do |dir_path| - if remove_flag - if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".color(:red) + if remove_flag + if FileUtils.rm_rf dir_path + puts "Removed...#{dir_path}".color(:red) + else + puts "Cannot remove #{dir_path}".color(:red) + end else - puts "Cannot remove #{dir_path}".color(:red) + puts "Can be removed: #{dir_path}".color(:red) end - else - puts "Can be removed: #{dir_path}".color(:red) end end @@ -48,20 +48,21 @@ namespace :gitlab do warn_user_is_not_gitlab move_suffix = "+orphaned+#{Time.now.to_i}" - repo_root = Gitlab.config.gitlab_shell.repos_path - # Look for global repos (legacy, depth 1) and normal repos (depth 2) - IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| - find.each_line do |path| - path.chomp! - repo_with_namespace = path. - sub(repo_root, ''). - sub(%r{^/*}, ''). - chomp('.git'). - chomp('.wiki') - next if Project.find_with_namespace(repo_with_namespace) - new_path = path + move_suffix - puts path.inspect + ' -> ' + new_path.inspect - File.rename(path, new_path) + Gitlab.config.repositories.storages.each do |name, repo_root| + # Look for global repos (legacy, depth 1) and normal repos (depth 2) + IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| + find.each_line do |path| + path.chomp! + repo_with_namespace = path. + sub(repo_root, ''). + sub(%r{^/*}, ''). + chomp('.git'). + chomp('.wiki') + next if Project.find_with_namespace(repo_with_namespace) + new_path = path + move_suffix + puts path.inspect + ' -> ' + new_path.inspect + File.rename(path, new_path) + end end end end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 4753f00c26a..dbdd4e977e8 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -2,73 +2,73 @@ namespace :gitlab do namespace :import do # How to use: # - # 1. copy the bare repos under the repos_path (commonly /home/git/repositories) + # 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories) # 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production # # Notes: # * The project owner will set to the first administator of the system # * Existing projects will be skipped # - desc "GitLab | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance" + desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do + Gitlab.config.repositories.storages.each do |name, git_base_path| + repos_to_import = Dir.glob(git_base_path + '/**/*.git') - git_base_path = Gitlab.config.gitlab_shell.repos_path - repos_to_import = Dir.glob(git_base_path + '/**/*.git') + repos_to_import.each do |repo_path| + # strip repo base path + repo_path[0..git_base_path.length] = '' - repos_to_import.each do |repo_path| - # strip repo base path - repo_path[0..git_base_path.length] = '' + path = repo_path.sub(/\.git$/, '') + group_name, name = File.split(path) + group_name = nil if group_name == '.' - path = repo_path.sub(/\.git$/, '') - group_name, name = File.split(path) - group_name = nil if group_name == '.' + puts "Processing #{repo_path}".color(:yellow) - puts "Processing #{repo_path}".color(:yellow) - - if path.end_with?('.wiki') - puts " * Skipping wiki repo" - next - end + if path.end_with?('.wiki') + puts " * Skipping wiki repo" + next + end - project = Project.find_with_namespace(path) + project = Project.find_with_namespace(path) - if project - puts " * #{project.name} (#{repo_path}) exists" - else - user = User.admins.reorder("id").first + if project + puts " * #{project.name} (#{repo_path}) exists" + else + user = User.admins.reorder("id").first - project_params = { - name: name, - path: name - } + project_params = { + name: name, + path: name + } - # find group namespace - if group_name - group = Namespace.find_by(path: group_name) - # create group namespace - unless group - group = Group.new(:name => group_name) - group.path = group_name - group.owner = user - if group.save - puts " * Created Group #{group.name} (#{group.id})".color(:green) - else - puts " * Failed trying to create group #{group.name}".color(:red) + # find group namespace + if group_name + group = Namespace.find_by(path: group_name) + # create group namespace + unless group + group = Group.new(:name => group_name) + group.path = group_name + group.owner = user + if group.save + puts " * Created Group #{group.name} (#{group.id})".color(:green) + else + puts " * Failed trying to create group #{group.name}".color(:red) + end end + # set project group + project_params[:namespace_id] = group.id end - # set project group - project_params[:namespace_id] = group.id - end - project = Projects::CreateService.new(user, project_params).execute + project = Projects::CreateService.new(user, project_params).execute - if project.persisted? - puts " * Created #{project.name} (#{repo_path})".color(:green) - project.update_repository_size - project.update_commit_count - else - puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) - puts " Errors: #{project.errors.messages}".color(:red) + if project.persisted? + puts " * Created #{project.name} (#{repo_path})".color(:green) + project.update_repository_size + project.update_commit_count + else + puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) + puts " Errors: #{project.errors.messages}".color(:red) + end end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 352b566df24..fe43d40e6d2 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -62,7 +62,10 @@ namespace :gitlab do puts "" puts "GitLab Shell".color(:yellow) puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" - puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}" + puts "Repository storage paths:" + Gitlab.config.repositories.storages.each do |name, path| + puts "- #{name}: \t#{path}" + end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index c7596e7abcb..ffcc76e5498 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -9,7 +9,7 @@ namespace :gitlab do scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end scope.find_each do |project| - base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) + base = File.join(project.repository_storage_path, project.path_with_namespace) puts base + '.git' puts base + '.wiki.git' end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index b1648a4602a..263798e9c22 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -12,7 +12,6 @@ namespace :gitlab do gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') - repos_path = Gitlab.config.gitlab_shell.repos_path target_dir = Gitlab.config.gitlab_shell.path # Clone if needed @@ -35,7 +34,6 @@ namespace :gitlab do user: user, gitlab_url: gitlab_url, http_settings: {self_signed_cert: false}.stringify_keys, - repos_path: repos_path, auth_file: File.join(home_dir, ".ssh", "authorized_keys"), redis: { bin: %x{which redis-cli}.chomp, @@ -58,10 +56,10 @@ namespace :gitlab do File.open("config.yml", "w+") {|f| f.puts config.to_yaml} # Launch installation process - system(*%W(bin/install)) + system(*%W(bin/install) + repository_storage_paths_args) # (Re)create hooks - system(*%W(bin/create-hooks)) + system(*%W(bin/create-hooks) + repository_storage_paths_args) end # Required for debian packaging with PKGR: Setup .ssh/environment with @@ -87,7 +85,8 @@ namespace :gitlab do if File.exists?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.path_with_namespace) + if Gitlab::Shell.new.add_repository(project.repository_storage_path, + project.path_with_namespace) print '.' else print 'F' @@ -138,4 +137,3 @@ namespace :gitlab do system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) end end - diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index d0c019044b7..ab96b1d3593 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -125,10 +125,16 @@ namespace :gitlab do end def all_repos - IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp + Gitlab.config.repositories.storages.each do |name, path| + IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end end end end + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end |