diff options
author | Alejandro RodrÃguez <alejorro70@gmail.com> | 2016-06-14 12:27:33 -0400 |
---|---|---|
committer | Alejandro RodrÃguez <alejorro70@gmail.com> | 2016-06-14 12:32:51 -0400 |
commit | 98c5e6295b201eeaa760d104f9b43e237f1dbd49 (patch) | |
tree | 9769f7903dabe8b5f8745ced9eb9f1bcf6d7fee4 | |
parent | c6cc7680e7da89c145b7e2075266791a3f594eec (diff) | |
download | gitlab-ce-98c5e6295b201eeaa760d104f9b43e237f1dbd49.tar.gz |
Refactor repository paths handling to allow multiple git mount pointsshards-test
42 files changed, 449 insertions, 328 deletions
diff --git a/CHANGELOG b/CHANGELOG index 74fb52d3aeb..c054c11824d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -79,6 +79,7 @@ v 8.9.0 (unreleased) - Allow users to create confidential issues in private projects - Measure CPU time for instrumented methods - Instrument private methods and private instance methods by default instead just public methods + - Refactor repository paths handling to allow multiple git mount points v 8.8.5 (unreleased) - Ensure branch cleanup regardless of whether the GitHub import process succeeds diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION index 4a36342fcab..fd2a01863fd 100644 --- a/GITLAB_SHELL_VERSION +++ b/GITLAB_SHELL_VERSION @@ -1 +1 @@ -3.0.0 +3.1.0 diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index d30dd66202b..825dce30863 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -323,9 +323,9 @@ module ProjectsHelper end end - def sanitize_repo_path(message) + def sanitize_repo_path(project, message) return '' unless message.present? - message.strip.gsub(Gitlab.config.gitlab_shell.repos_path.chomp('/'), "[REPOS PATH]") + message.strip.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]") end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index da19462f265..9c0594b1e06 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -21,8 +21,10 @@ class Namespace < ActiveRecord::Base delegate :name, to: :owner, allow_nil: true, prefix: true - after_create :ensure_dir_exist after_update :move_dir, if: :path_changed? + + # Save the storage paths before the projects are destroyed to use them on after destroy + before_destroy(prepend: true) { @old_repository_storage_paths = repository_storage_paths } after_destroy :rm_dir scope :root, -> { where('type IS NULL') } @@ -87,51 +89,35 @@ class Namespace < ActiveRecord::Base owner_name end - def ensure_dir_exist - gitlab_shell.add_namespace(path) - end - - def rm_dir - # Move namespace directory into trash. - # We will remove it later async - new_path = "#{path}+#{id}+deleted" - - if gitlab_shell.mv_namespace(path, new_path) - message = "Namespace directory \"#{path}\" moved to \"#{new_path}\"" - Gitlab::AppLogger.info message - - # Remove namespace directroy async with delay so - # GitLab has time to remove all projects first - GitlabShellWorker.perform_in(5.minutes, :rm_namespace, new_path) - end - end - def move_dir - # Ensure old directory exists before moving it - gitlab_shell.add_namespace(path_was) - if any_project_has_container_registry_tags? raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry') end - if gitlab_shell.mv_namespace(path_was, path) - Gitlab::UploadsTransfer.new.rename_namespace(path_was, path) - - # If repositories moved successfully we need to - # send update instructions to users. - # However we cannot allow rollback since we moved namespace dir - # So we basically we mute exceptions in next actions - begin - send_update_instructions - rescue - # Returning false does not rollback after_* transaction but gives - # us information about failing some of tasks - false + # Move the namespace directory in all storages paths used by member projects + repository_storage_paths.each do |repository_storage_path| + # Ensure old directory exists before moving it + gitlab_shell.add_namespace(repository_storage_path, path_was) + + unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path) + # if we cannot move namespace directory we should rollback + # db changes in order to prevent out of sync between db and fs + raise Exception.new('namespace directory cannot be moved') end - else - # if we cannot move namespace directory we should rollback - # db changes in order to prevent out of sync between db and fs - raise Exception.new('namespace directory cannot be moved') + end + + Gitlab::UploadsTransfer.new.rename_namespace(path_was, path) + + # If repositories moved successfully we need to + # send update instructions to users. + # However we cannot allow rollback since we moved namespace dir + # So we basically we mute exceptions in next actions + begin + send_update_instructions + rescue + # Returning false does not rollback after_* transaction but gives + # us information about failing some of tasks + false end end @@ -152,4 +138,28 @@ class Namespace < ActiveRecord::Base def find_fork_of(project) projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id) end + + private + + def repository_storage_paths + projects.to_a.map(&:repository_storage_path).uniq + end + + def rm_dir + # Remove the namespace directory in all storages paths used by member projects + @old_repository_storage_paths.each do |repository_storage_path| + # Move namespace directory into trash. + # We will remove it later async + new_path = "#{path}+#{id}+deleted" + + if gitlab_shell.mv_namespace(repository_storage_path, path, new_path) + message = "Namespace directory \"#{path}\" moved to \"#{new_path}\"" + Gitlab::AppLogger.info message + + # Remove namespace directroy async with delay so + # GitLab has time to remove all projects first + GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path) + end + end + end end diff --git a/app/models/project.rb b/app/models/project.rb index 0d2e612436a..4ea3b09c0c9 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -26,6 +26,15 @@ class Project < ActiveRecord::Base default_value_for :container_registry_enabled, gitlab_config_features.container_registry default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled } + # TODO: It used to be a namespace task to ensure the namespace directory + # existance, but since now projects might be in different paths (shards) they + # must account for that. However, checking the directory exists every time the + # project is saved might be inefficient. Consider moving this logic to another + # moment, or perhaps not to assume the directory exists and have gitlab-shell + # take that into consideration (doing `mkdir` himself before each operation, + # for example) + after_save :ensure_dir_exist + # set last_activity_at to the same as created_at after_create :set_last_activity_at def set_last_activity_at @@ -351,6 +360,14 @@ class Project < ActiveRecord::Base end end + def repository_storage + super || 'default' + end + + def repository_storage_path + Gitlab.config.repositories.storages[repository_storage] + end + def team @team ||= ProjectTeam.new(self) end @@ -803,12 +820,12 @@ class Project < ActiveRecord::Base raise Exception.new('Project cannot be renamed, because tags are present in its container registry') end - if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace) + if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) # If repository moved successfully we need to send update instructions to users. # However we cannot allow rollback since we moved repository # So we basically we mute exceptions in next actions begin - gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") + gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") send_move_instructions(old_path_with_namespace) reset_events_cache @@ -949,7 +966,7 @@ class Project < ActiveRecord::Base def create_repository # Forked import is handled asynchronously unless forked? - if gitlab_shell.add_repository(path_with_namespace) + if gitlab_shell.add_repository(repository_storage_path, path_with_namespace) repository.after_create true else @@ -1078,4 +1095,8 @@ class Project < ActiveRecord::Base ensure @errors = original_errors end + + def ensure_dir_exist + gitlab_shell.add_namespace(repository_storage_path, namespace.path) + end end diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 25d82929c0b..a255710f577 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -159,7 +159,7 @@ class ProjectWiki private def init_repo(path_with_namespace) - gitlab_shell.add_repository(path_with_namespace) + gitlab_shell.add_repository(project.repository_storage_path, path_with_namespace) end def commit_details(action, message = nil, title = nil) @@ -173,7 +173,7 @@ class ProjectWiki end def path_to_repo - @path_to_repo ||= File.join(Gitlab.config.gitlab_shell.repos_path, "#{path_with_namespace}.git") + @path_to_repo ||= File.join(project.repository_storage_path, "#{path_with_namespace}.git") end def update_project_activity diff --git a/app/models/repository.rb b/app/models/repository.rb index 1ab163510bf..bb15be3a913 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -39,7 +39,7 @@ class Repository # Return absolute path to repository def path_to_repo @path_to_repo ||= File.expand_path( - File.join(Gitlab.config.gitlab_shell.repos_path, path_with_namespace + ".git") + File.join(@project.repository_storage_path, path_with_namespace + ".git") ) end diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index f09072975c3..882606e38d0 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -51,13 +51,13 @@ module Projects return true if params[:skip_repo] == true # There is a possibility project does not have repository or wiki - return true unless gitlab_shell.exists?(path + '.git') + return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git') new_path = removal_path(path) - if gitlab_shell.mv_repository(path, new_path) + if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path) log_info("Repository \"#{path}\" moved to \"#{new_path}\"") - GitlabShellWorker.perform_in(5.minutes, :remove_repository, new_path) + GitlabShellWorker.perform_in(5.minutes, :remove_repository, project.repository_storage_path, new_path) else false end diff --git a/app/services/projects/housekeeping_service.rb b/app/services/projects/housekeeping_service.rb index 43db29315a1..a47df22f1ba 100644 --- a/app/services/projects/housekeeping_service.rb +++ b/app/services/projects/housekeeping_service.rb @@ -24,7 +24,7 @@ module Projects def execute raise LeaseTaken unless try_obtain_lease - GitlabShellOneShotWorker.perform_async(:gc, @project.path_with_namespace) + GitlabShellOneShotWorker.perform_async(:gc, @project.repository_storage_path, @project.path_with_namespace) ensure Gitlab::Metrics.measure(:reset_pushes_since_gc) do @project.update_column(:pushes_since_gc, 0) diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb index c4838d31f2f..70815b2dd5b 100644 --- a/app/services/projects/import_service.rb +++ b/app/services/projects/import_service.rb @@ -37,7 +37,7 @@ module Projects def import_repository begin - gitlab_shell.import_repository(project.path_with_namespace, project.import_url) + gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url) rescue Gitlab::Shell::Error => e raise Error, "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}" end diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 03b57dea51e..bc7f8bf433b 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -50,12 +50,12 @@ module Projects project.send_move_instructions(old_path) # Move main repository - unless gitlab_shell.mv_repository(old_path, new_path) + unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, new_path) raise TransferError.new('Cannot move project') end # Move wiki repo also if present - gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki") + gitlab_shell.mv_repository(project.repository_storage_path, "#{old_path}.wiki", "#{new_path}.wiki") # clear project cached events project.reset_events_cache diff --git a/app/views/projects/imports/new.html.haml b/app/views/projects/imports/new.html.haml index a8a8caf7280..2cd8d03e30e 100644 --- a/app/views/projects/imports/new.html.haml +++ b/app/views/projects/imports/new.html.haml @@ -10,7 +10,7 @@ .panel-body %pre :preserve - #{sanitize_repo_path(@project.import_error)} + #{sanitize_repo_path(@project, @project.import_error)} = form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f| = render "shared/import_form", f: f diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index f3327ca9e61..09035a7cf2d 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -4,10 +4,10 @@ class PostReceive sidekiq_options queue: :post_receive def perform(repo_path, identifier, changes) - if repo_path.start_with?(Gitlab.config.gitlab_shell.repos_path.to_s) - repo_path.gsub!(Gitlab.config.gitlab_shell.repos_path.to_s, "") + if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) } + repo_path.gsub!(path[1].to_s, "") else - log("Check gitlab.yml config for correct gitlab_shell.repos_path variable. \"#{Gitlab.config.gitlab_shell.repos_path}\" does not match \"#{repo_path}\"") + log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"") end post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes) diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index d947f105516..f7604e48f83 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -12,7 +12,7 @@ class RepositoryForkWorker return end - result = gitlab_shell.fork_repository(source_path, target_path) + result = gitlab_shell.fork_repository(project.repository_storage_path, source_path, target_path) unless result logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}") project.mark_import_as_failed('The project could not be forked.') diff --git a/config/gitlab.teatro.yml b/config/gitlab.teatro.yml index 01c8dc5ff98..75b79b837e0 100644 --- a/config/gitlab.teatro.yml +++ b/config/gitlab.teatro.yml @@ -47,11 +47,13 @@ production: &base backup: path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) + repositories: + storages: # REPO PATHS MUST NOT BE A SYMLINK!!! + default: /apps/repositories/ + gitlab_shell: path: /apps/gitlab-shell/ - # REPOS_PATH MUST NOT BE A SYMLINK!!! - repos_path: /apps/repositories/ hooks_path: /apps/gitlab-shell/hooks/ upload_pack: true diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 75e1a3c1093..7447f6a5cdd 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -428,6 +428,12 @@ production: &base satellites: path: /home/git/gitlab-satellites/ + ## Repositories settings + repositories: + # Paths where repositories can be stored. Give the canonicalized absolute pathname. + # NOTE: REPOS PATHS MUST NOT CONTAIN ANY SYMLINK!!! + storages: # You must have at least a `default` storage path. + ## Backup settings backup: path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) @@ -452,9 +458,6 @@ production: &base ## GitLab Shell settings gitlab_shell: path: /home/git/gitlab-shell/ - - # REPOS_PATH MUST NOT BE A SYMLINK!!! - repos_path: /home/git/repositories/ hooks_path: /home/git/gitlab-shell/hooks/ # File that contains the secret key for verifying access for gitlab-shell. @@ -528,11 +531,13 @@ test: # user: YOUR_USERNAME satellites: path: tmp/tests/gitlab-satellites/ + repositories: + storages: + default: tmp/tests/repositories/ backup: path: tmp/tests/backups gitlab_shell: path: tmp/tests/gitlab-shell/ - repos_path: tmp/tests/repositories/ hooks_path: tmp/tests/gitlab-shell/hooks/ issues_tracker: redmine: diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 916fd33e767..dab657d1cc4 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -293,6 +293,11 @@ Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker' # +# Repositories +# +Settings.repositories.storages['default'] ||= Settings.gitlab['user_home'] + '/repositories/' + +# # GitLab Shell # Settings['gitlab_shell'] ||= Settingslogic.new({}) @@ -301,7 +306,6 @@ Settings.gitlab_shell['hooks_path'] ||= Settings.gitlab['user_home'] + '/gitla Settings.gitlab_shell['secret_file'] ||= Rails.root.join('.gitlab_shell_secret') Settings.gitlab_shell['receive_pack'] = true if Settings.gitlab_shell['receive_pack'].nil? Settings.gitlab_shell['upload_pack'] = true if Settings.gitlab_shell['upload_pack'].nil? -Settings.gitlab_shell['repos_path'] ||= Settings.gitlab['user_home'] + '/repositories/' Settings.gitlab_shell['ssh_host'] ||= Settings.gitlab.ssh_host Settings.gitlab_shell['ssh_port'] ||= 22 Settings.gitlab_shell['ssh_user'] ||= Settings.gitlab.user diff --git a/db/migrate/20160608195742_add_repository_storage_to_projects.rb b/db/migrate/20160608195742_add_repository_storage_to_projects.rb new file mode 100644 index 00000000000..23d043e90a9 --- /dev/null +++ b/db/migrate/20160608195742_add_repository_storage_to_projects.rb @@ -0,0 +1,5 @@ +class AddRepositoryStorageToProjects < ActiveRecord::Migration + def change + add_column :projects, :repository_storage, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index 5fe39c5e59c..cd3ae15ab7f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -779,6 +779,7 @@ ActiveRecord::Schema.define(version: 20160610301627) do t.boolean "last_repository_check_failed" t.datetime "last_repository_check_at" t.boolean "container_registry_enabled" + t.string "repository_storage" t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false t.boolean "has_external_issue_tracker" end diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md index 8a38937062e..2b305cb5c99 100644 --- a/doc/raketasks/import.md +++ b/doc/raketasks/import.md @@ -14,7 +14,8 @@ - For omnibus-gitlab, it is located at: `/var/opt/gitlab/git-data/repositories` by default, unless you changed it in the `/etc/gitlab/gitlab.rb` file. - For installations from source, it is usually located at: `/home/git/repositories` or you can see where -your repositories are located by looking at `config/gitlab.yml` under the `gitlab_shell => repos_path` entry. +your repositories are located by looking at `config/gitlab.yml` under the `repositories => storages` entries +(you'll usually use the `default` storage path to start). New folder needs to have git user ownership and read/write/execute access for git user and its group: diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 3ac7b50c4ce..de82783e3f3 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -20,6 +20,20 @@ module API @wiki ||= params[:project].end_with?('.wiki') && !Project.find_with_namespace(params[:project]) end + + def project + @project ||= begin + project_path = params[:project] + + # Check for *.wiki repositories. + # Strip out the .wiki from the pathname before finding the + # project. This applies the correct project permissions to + # the wiki repository as well. + project_path.chomp!('.wiki') if wiki? + + Project.find_with_namespace(project_path) + end + end end post "/allowed" do @@ -34,16 +48,6 @@ module API User.find_by(id: params[:user_id]) end - project_path = params[:project] - - # Check for *.wiki repositories. - # Strip out the .wiki from the pathname before finding the - # project. This applies the correct project permissions to - # the wiki repository as well. - project_path.chomp!('.wiki') if wiki? - - project = Project.find_with_namespace(project_path) - access = if wiki? Gitlab::GitAccessWiki.new(actor, project) @@ -51,7 +55,17 @@ module API Gitlab::GitAccess.new(actor, project) end - access.check(params[:action], params[:changes]) + access_status = access.check(params[:action], params[:changes]) + + response = { status: access_status.status, message: access_status.message } + + if access_status.status + # Return the repository storage path so that gitlab-shell has it when + # handling ssh commands + response[:repository_storage_path] = project.repository_storage_path + end + + response end # diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 7b91215d50b..9686b0b6f40 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -2,8 +2,6 @@ require 'yaml' module Backup class Repository - attr_reader :repos_path - def dump prepare @@ -50,10 +48,12 @@ module Backup end def restore - if File.exists?(repos_path) + Gitlab.config.repositories.storages.each do |name, path| + next unless File.exists?(path) + # Move repos dir to 'repositories.old' dir - bk_repos_path = File.join(repos_path, '..', 'repositories.old.' + Time.now.to_i.to_s) - FileUtils.mv(repos_path, bk_repos_path) + bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) + FileUtils.mv(path, bk_repos_path) end FileUtils.mkdir_p(repos_path) @@ -61,7 +61,7 @@ module Backup Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " - project.namespace.ensure_dir_exist if project.namespace + project.ensure_dir_exist if File.exists?(path_to_bundle(project)) FileUtils.mkdir_p(path_to_repo(project)) @@ -120,10 +120,6 @@ module Backup File.join(backup_repos_path, project.path_with_namespace + ".bundle") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - def backup_repos_path File.join(Gitlab.config.backup.path, "repositories") end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 3e3986d6382..e31840ef919 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -18,77 +18,82 @@ module Gitlab # Init new repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # add_repository("gitlab/gitlab-ci") + # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - def add_repository(name) + def add_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'add-project', "#{name}.git"]) + 'add-project', storage, "#{name}.git"]) end # Import repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") + # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # - def import_repository(name, url) - output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900']) + def import_repository(storage, name, url) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', + storage, "#{name}.git", url, '900']) raise Error, output unless status.zero? true end # Move repository - # + # storage - project's storage path # path - project path with namespace # new_path - new project path with namespace # # Ex. - # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new") + # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - def mv_repository(path, new_path) + def mv_repository(storage, path, new_path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', - "#{path}.git", "#{new_path}.git"]) + storage, "#{path}.git", "#{new_path}.git"]) end # Fork repository to new namespace - # + # storage - project's storage path # path - project path with namespace # fork_namespace - namespace for forked project # # Ex. - # fork_repository("gitlab/gitlab-ci", "randx") + # fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx") # - def fork_repository(path, fork_namespace) + def fork_repository(storage, path, fork_namespace) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', - "#{path}.git", fork_namespace]) + storage, "#{path}.git", fork_namespace]) end # Remove repository from file system # + # storage - project's storage path # name - project path with namespace # # Ex. - # remove_repository("gitlab/gitlab-ci") + # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - def remove_repository(name) + def remove_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'rm-project', "#{name}.git"]) + 'rm-project', storage, "#{name}.git"]) end # Gc repository # + # storage - project storage path # path - project path with namespace # # Ex. - # gc("gitlab/gitlab-ci") + # gc("/path/to/storage", "gitlab/gitlab-ci") # - def gc(path) + def gc(storage, path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', - "#{path}.git"]) + storage, "#{path}.git"]) end # Add new key to gitlab-shell @@ -133,31 +138,31 @@ module Gitlab # Add empty directory for storing repositories # # Ex. - # add_namespace("gitlab") + # add_namespace("/path/to/storage", "gitlab") # - def add_namespace(name) - FileUtils.mkdir(full_path(name), mode: 0770) unless exists?(name) + def add_namespace(storage, name) + FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name) end # Remove directory from repositories storage # Every repository inside this directory will be removed too # # Ex. - # rm_namespace("gitlab") + # rm_namespace("/path/to/storage", "gitlab") # - def rm_namespace(name) - FileUtils.rm_r(full_path(name), force: true) + def rm_namespace(storage, name) + FileUtils.rm_r(full_path(storage, name), force: true) end # Move namespace directory inside repositories storage # # Ex. - # mv_namespace("gitlab", "gitlabhq") + # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - def mv_namespace(old_name, new_name) - return false if exists?(new_name) || !exists?(old_name) + def mv_namespace(storage, old_name, new_name) + return false if exists?(storage, new_name) || !exists?(storage, old_name) - FileUtils.mv(full_path(old_name), full_path(new_name)) + FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) end def url_to_repo(path) @@ -176,11 +181,11 @@ module Gitlab # Check if such directory exists in repositories. # # Usage: - # exists?('gitlab') - # exists?('gitlab/cookies.git') + # exists?(storage, 'gitlab') + # exists?(storage, 'gitlab/cookies.git') # - def exists?(dir_name) - File.exist?(full_path(dir_name)) + def exists?(storage, dir_name) + File.exist?(full_path(storage, dir_name)) end protected @@ -193,14 +198,10 @@ module Gitlab File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - - def full_path(dir_name) + def full_path(storage, dir_name) raise ArgumentError.new("Directory name can't be blank") if dir_name.blank? - File.join(repos_path, dir_name) + File.join(storage, dir_name) end def gitlab_shell_projects_path diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index e5cf66a0371..85be1361a08 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -154,7 +154,7 @@ module Gitlab def import_wiki unless project.wiki_enabled? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) project.update_attribute(:wiki_enabled, true) end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 12d6ac45fb6..95a9d393ae7 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -356,97 +356,108 @@ namespace :gitlab do ######################## def check_repo_base_exists - print "Repo base directory exists? ... " + puts "Repo base directory exists?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.exists?(repo_base_path) - puts "yes".color(:green) - else - puts "no".color(:red) - puts "#{repo_base_path} is missing".color(:red) - try_fixing_it( - "This should have been created when setting up GitLab Shell.", - "Make sure it's set correctly in config/gitlab.yml", - "Make sure GitLab Shell is installed correctly." - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + if File.exists?(repo_base_path) + puts "yes".color(:green) + else + puts "no".color(:red) + puts "#{repo_base_path} is missing".color(:red) + try_fixing_it( + "This should have been created when setting up GitLab Shell.", + "Make sure it's set correctly in config/gitlab.yml", + "Make sure GitLab Shell is installed correctly." + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_is_not_symlink - print "Repo base directory is a symlink? ... " + puts "Repo storage directories are symlinks?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - unless File.symlink?(repo_base_path) - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Make sure it's set to the real directory in config/gitlab.yml" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + unless File.symlink?(repo_base_path) + puts "no".color(:green) + else + puts "yes".color(:red) + try_fixing_it( + "Make sure it's set to the real directory in config/gitlab.yml" + ) + fix_and_rerun + end end end def check_repo_base_permissions - print "Repo base access is drwxrws---? ... " + puts "Repo paths access is drwxrws---?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", - "sudo chmod -R ug-s #{repo_base_path}", - "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") + puts "yes".color(:green) + else + puts "no".color(:red) + try_fixing_it( + "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", + "sudo chmod -R ug-s #{repo_base_path}", + "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_user_and_group gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group - print "Repo base owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}? ... " + puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - uid = uid_for(gitlab_shell_ssh_user) - gid = gid_for(gitlab_shell_owner_group) - if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid - puts "yes".color(:green) - else - puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) - try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + uid = uid_for(gitlab_shell_ssh_user) + gid = gid_for(gitlab_shell_owner_group) + if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid + puts "yes".color(:green) + else + puts "no".color(:red) + puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) + try_fixing_it( + "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end @@ -785,13 +796,13 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - namespace_dirs = Dir.glob( - File.join(Gitlab.config.gitlab_shell.repos_path, '*') - ) + Gitlab.config.repositories.storages.each do |name, path| + namespace_dirs = Dir.glob(File.join(path, '*')) - namespace_dirs.each do |namespace_dir| - repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + namespace_dirs.each do |namespace_dir| + repo_dirs = Dir.glob(File.join(namespace_dir, '*')) + repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + end end end end @@ -799,12 +810,12 @@ namespace :gitlab do namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| - username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue)) + username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| File.join( - Gitlab.config.gitlab_shell.repos_path, + p.repository_storage_path, "#{p.path_with_namespace}.git" ) end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index ab0028d6603..b7cbdc6cd78 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -5,36 +5,36 @@ namespace :gitlab do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] - namespaces = Namespace.pluck(:path) - git_base_path = Gitlab.config.gitlab_shell.repos_path - all_dirs = Dir.glob(git_base_path + '/*') + Gitlab.config.repositories.storages.each do |name, git_base_path| + all_dirs = Dir.glob(git_base_path + '/*') - puts git_base_path.color(:yellow) - puts "Looking for directories to remove... " + puts git_base_path.color(:yellow) + puts "Looking for directories to remove... " - all_dirs.reject! do |dir| - # skip if git repo - dir =~ /.git$/ - end + all_dirs.reject! do |dir| + # skip if git repo + dir =~ /.git$/ + end - all_dirs.reject! do |dir| - dir_name = File.basename dir + all_dirs.reject! do |dir| + dir_name = File.basename dir - # skip if namespace present - namespaces.include?(dir_name) - end + # skip if namespace present + namespaces.include?(dir_name) + end - all_dirs.each do |dir_path| + all_dirs.each do |dir_path| - if remove_flag - if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".color(:red) + if remove_flag + if FileUtils.rm_rf dir_path + puts "Removed...#{dir_path}".color(:red) + else + puts "Cannot remove #{dir_path}".color(:red) + end else - puts "Cannot remove #{dir_path}".color(:red) + puts "Can be removed: #{dir_path}".color(:red) end - else - puts "Can be removed: #{dir_path}".color(:red) end end @@ -48,20 +48,21 @@ namespace :gitlab do warn_user_is_not_gitlab move_suffix = "+orphaned+#{Time.now.to_i}" - repo_root = Gitlab.config.gitlab_shell.repos_path - # Look for global repos (legacy, depth 1) and normal repos (depth 2) - IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| - find.each_line do |path| - path.chomp! - repo_with_namespace = path. - sub(repo_root, ''). - sub(%r{^/*}, ''). - chomp('.git'). - chomp('.wiki') - next if Project.find_with_namespace(repo_with_namespace) - new_path = path + move_suffix - puts path.inspect + ' -> ' + new_path.inspect - File.rename(path, new_path) + Gitlab.config.repositories.storages.each do |name, repo_root| + # Look for global repos (legacy, depth 1) and normal repos (depth 2) + IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| + find.each_line do |path| + path.chomp! + repo_with_namespace = path. + sub(repo_root, ''). + sub(%r{^/*}, ''). + chomp('.git'). + chomp('.wiki') + next if Project.find_with_namespace(repo_with_namespace) + new_path = path + move_suffix + puts path.inspect + ' -> ' + new_path.inspect + File.rename(path, new_path) + end end end end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 4753f00c26a..dbdd4e977e8 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -2,73 +2,73 @@ namespace :gitlab do namespace :import do # How to use: # - # 1. copy the bare repos under the repos_path (commonly /home/git/repositories) + # 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories) # 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production # # Notes: # * The project owner will set to the first administator of the system # * Existing projects will be skipped # - desc "GitLab | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance" + desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do + Gitlab.config.repositories.storages.each do |name, git_base_path| + repos_to_import = Dir.glob(git_base_path + '/**/*.git') - git_base_path = Gitlab.config.gitlab_shell.repos_path - repos_to_import = Dir.glob(git_base_path + '/**/*.git') + repos_to_import.each do |repo_path| + # strip repo base path + repo_path[0..git_base_path.length] = '' - repos_to_import.each do |repo_path| - # strip repo base path - repo_path[0..git_base_path.length] = '' + path = repo_path.sub(/\.git$/, '') + group_name, name = File.split(path) + group_name = nil if group_name == '.' - path = repo_path.sub(/\.git$/, '') - group_name, name = File.split(path) - group_name = nil if group_name == '.' + puts "Processing #{repo_path}".color(:yellow) - puts "Processing #{repo_path}".color(:yellow) - - if path.end_with?('.wiki') - puts " * Skipping wiki repo" - next - end + if path.end_with?('.wiki') + puts " * Skipping wiki repo" + next + end - project = Project.find_with_namespace(path) + project = Project.find_with_namespace(path) - if project - puts " * #{project.name} (#{repo_path}) exists" - else - user = User.admins.reorder("id").first + if project + puts " * #{project.name} (#{repo_path}) exists" + else + user = User.admins.reorder("id").first - project_params = { - name: name, - path: name - } + project_params = { + name: name, + path: name + } - # find group namespace - if group_name - group = Namespace.find_by(path: group_name) - # create group namespace - unless group - group = Group.new(:name => group_name) - group.path = group_name - group.owner = user - if group.save - puts " * Created Group #{group.name} (#{group.id})".color(:green) - else - puts " * Failed trying to create group #{group.name}".color(:red) + # find group namespace + if group_name + group = Namespace.find_by(path: group_name) + # create group namespace + unless group + group = Group.new(:name => group_name) + group.path = group_name + group.owner = user + if group.save + puts " * Created Group #{group.name} (#{group.id})".color(:green) + else + puts " * Failed trying to create group #{group.name}".color(:red) + end end + # set project group + project_params[:namespace_id] = group.id end - # set project group - project_params[:namespace_id] = group.id - end - project = Projects::CreateService.new(user, project_params).execute + project = Projects::CreateService.new(user, project_params).execute - if project.persisted? - puts " * Created #{project.name} (#{repo_path})".color(:green) - project.update_repository_size - project.update_commit_count - else - puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) - puts " Errors: #{project.errors.messages}".color(:red) + if project.persisted? + puts " * Created #{project.name} (#{repo_path})".color(:green) + project.update_repository_size + project.update_commit_count + else + puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) + puts " Errors: #{project.errors.messages}".color(:red) + end end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 352b566df24..fe43d40e6d2 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -62,7 +62,10 @@ namespace :gitlab do puts "" puts "GitLab Shell".color(:yellow) puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" - puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}" + puts "Repository storage paths:" + Gitlab.config.repositories.storages.each do |name, path| + puts "- #{name}: \t#{path}" + end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index c7596e7abcb..ffcc76e5498 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -9,7 +9,7 @@ namespace :gitlab do scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end scope.find_each do |project| - base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) + base = File.join(project.repository_storage_path, project.path_with_namespace) puts base + '.git' puts base + '.wiki.git' end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index b1648a4602a..00f0234923a 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -12,7 +12,6 @@ namespace :gitlab do gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') - repos_path = Gitlab.config.gitlab_shell.repos_path target_dir = Gitlab.config.gitlab_shell.path # Clone if needed @@ -35,7 +34,6 @@ namespace :gitlab do user: user, gitlab_url: gitlab_url, http_settings: {self_signed_cert: false}.stringify_keys, - repos_path: repos_path, auth_file: File.join(home_dir, ".ssh", "authorized_keys"), redis: { bin: %x{which redis-cli}.chomp, @@ -87,7 +85,8 @@ namespace :gitlab do if File.exists?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.path_with_namespace) + if Gitlab::Shell.new.add_repository(project.repository_storage_path, + project.path_with_namespace) print '.' else print 'F' @@ -138,4 +137,3 @@ namespace :gitlab do system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) end end - diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index d0c019044b7..68a099f8307 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -125,9 +125,11 @@ namespace :gitlab do end def all_repos - IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp + Gitlab.config.repositories.storages.each do |name, path| + IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end end end end diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index 09e0bbfd00b..604204cca0a 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -123,11 +123,17 @@ describe ProjectsHelper do end describe '#sanitized_import_error' do + let(:project) { create(:project) } + + before do + allow(project).to receive(:repository_storage_path).and_return('/base/repo/path') + end + it 'removes the repo path' do - repo = File.join(Gitlab.config.gitlab_shell.repos_path, '/namespace/test.git') + repo = '/base/repo/path/namespace/test.git' import_error = "Could not clone #{repo}\n" - expect(sanitize_repo_path(import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git') + expect(sanitize_repo_path(project, import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git') end end end diff --git a/spec/lib/gitlab/backend/shell_spec.rb b/spec/lib/gitlab/backend/shell_spec.rb index fd869f48b5c..e15f13f985b 100644 --- a/spec/lib/gitlab/backend/shell_spec.rb +++ b/spec/lib/gitlab/backend/shell_spec.rb @@ -13,6 +13,11 @@ describe Gitlab::Shell, lib: true do it { is_expected.to respond_to :add_repository } it { is_expected.to respond_to :remove_repository } it { is_expected.to respond_to :fork_repository } + it { is_expected.to respond_to :gc } + it { is_expected.to respond_to :add_namespace } + it { is_expected.to respond_to :rm_namespace } + it { is_expected.to respond_to :mv_namespace } + it { is_expected.to respond_to :exists? } it { expect(gitlab_shell.url_to_repo('diaspora')).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + "diaspora.git") } diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 4e68ac5e63a..cbea407f9bf 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -57,6 +57,7 @@ describe Namespace, models: true do describe :move_dir do before do @namespace = create :namespace + @project = create :project, namespace: @namespace allow(@namespace).to receive(:path_changed?).and_return(true) end @@ -87,8 +88,13 @@ describe Namespace, models: true do end describe :rm_dir do - it "should remove dir" do - expect(namespace.rm_dir).to be_truthy + let!(:project) { create(:project, namespace: namespace) } + let!(:path) { File.join(Gitlab.config.repositories.storages.default, namespace.path) } + + before { namespace.destroy } + + it "should remove its dirs when deleted" do + expect(File.exist?(path)).to be(false) end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 30aa2b70c8d..7b44c7160b8 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -108,6 +108,28 @@ describe Project, models: true do end end + describe '#repository_storage' do + let(:project) { create(:project, repository_storage: nil) } + + it 'should return the default repository storage if nil' do + expect(project.repository_storage).to eq('default') + end + end + + describe '#repository_storage_path' do + let(:project) { create(:project, repository_storage: 'custom') } + + before do + FileUtils.mkdir('tmp/tests/custom_repositories') + storages = { 'custom' => 'tmp/tests/custom_repositories' } + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + end + + it 'returns the repository storage path' do + expect(project.repository_storage_path).to eq('tmp/tests/custom_repositories') + end + end + it 'should return valid url to repo' do project = Project.new(path: 'somewhere') expect(project.url_to_repo).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + 'somewhere.git') @@ -690,12 +712,12 @@ describe Project, models: true do expect(gitlab_shell).to receive(:mv_repository). ordered. - with("#{ns}/foo", "#{ns}/#{project.path}"). + with(project.repository_storage_path, "#{ns}/foo", "#{ns}/#{project.path}"). and_return(true) expect(gitlab_shell).to receive(:mv_repository). ordered. - with("#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki"). + with(project.repository_storage_path, "#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki"). and_return(true) expect_any_instance_of(SystemHooksService). @@ -787,7 +809,7 @@ describe Project, models: true do context 'using a regular repository' do it 'creates the repository' do expect(shell).to receive(:add_repository). - with(project.path_with_namespace). + with(project.repository_storage_path, project.path_with_namespace). and_return(true) expect(project.repository).to receive(:after_create) @@ -797,7 +819,7 @@ describe Project, models: true do it 'adds an error if the repository could not be created' do expect(shell).to receive(:add_repository). - with(project.path_with_namespace). + with(project.repository_storage_path, project.path_with_namespace). and_return(false) expect(project.repository).not_to receive(:after_create) diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb index 22802dd0e05..097398442f4 100644 --- a/spec/requests/api/internal_spec.rb +++ b/spec/requests/api/internal_spec.rb @@ -72,6 +72,7 @@ describe API::API, api: true do expect(response.status).to eq(200) expect(json_response["status"]).to be_truthy + expect(json_response["repository_storage_path"]).to eq(project.repository_storage_path) end end @@ -81,6 +82,7 @@ describe API::API, api: true do expect(response.status).to eq(200) expect(json_response["status"]).to be_truthy + expect(json_response["repository_storage_path"]).to eq(project.repository_storage_path) end end end diff --git a/spec/services/destroy_group_service_spec.rb b/spec/services/destroy_group_service_spec.rb index afa89b84175..eca8ddd8ea4 100644 --- a/spec/services/destroy_group_service_spec.rb +++ b/spec/services/destroy_group_service_spec.rb @@ -23,8 +23,8 @@ describe DestroyGroupService, services: true do Sidekiq::Testing.inline! { destroy_group(group, user) } end - it { expect(gitlab_shell.exists?(group.path)).to be_falsey } - it { expect(gitlab_shell.exists?(remove_path)).to be_falsey } + it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey } + it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_falsey } end context 'Sidekiq fake' do @@ -33,8 +33,8 @@ describe DestroyGroupService, services: true do Sidekiq::Testing.fake! { destroy_group(group, user) } end - it { expect(gitlab_shell.exists?(group.path)).to be_falsey } - it { expect(gitlab_shell.exists?(remove_path)).to be_truthy } + it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey } + it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_truthy } end end diff --git a/spec/services/projects/housekeeping_service_spec.rb b/spec/services/projects/housekeeping_service_spec.rb index 4c5ced7e746..bd4dc6a0f79 100644 --- a/spec/services/projects/housekeeping_service_spec.rb +++ b/spec/services/projects/housekeeping_service_spec.rb @@ -12,7 +12,7 @@ describe Projects::HousekeepingService do it 'enqueues a sidekiq job' do expect(subject).to receive(:try_obtain_lease).and_return(true) - expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.path_with_namespace) + expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.repository_storage_path, project.path_with_namespace) subject.execute expect(project.pushes_since_gc).to eq(0) diff --git a/spec/services/projects/import_service_spec.rb b/spec/services/projects/import_service_spec.rb index 068c9a1219c..d5d4d7c56ef 100644 --- a/spec/services/projects/import_service_spec.rb +++ b/spec/services/projects/import_service_spec.rb @@ -36,7 +36,7 @@ describe Projects::ImportService, services: true do end it 'succeeds if repository import is successfully' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true) result = subject.execute @@ -44,7 +44,7 @@ describe Projects::ImportService, services: true do end it 'fails if repository import fails' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository')) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository')) result = subject.execute @@ -64,7 +64,7 @@ describe Projects::ImportService, services: true do end it 'succeeds if importer succeeds' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(true) result = subject.execute @@ -74,7 +74,7 @@ describe Projects::ImportService, services: true do it 'flushes various caches' do expect_any_instance_of(Gitlab::Shell).to receive(:import_repository). - with(project.path_with_namespace, project.import_url). + with(project.repository_storage_path, project.path_with_namespace, project.import_url). and_return(true) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute). @@ -90,7 +90,7 @@ describe Projects::ImportService, services: true do end it 'fails if importer fails' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(false) result = subject.execute @@ -100,7 +100,7 @@ describe Projects::ImportService, services: true do end it 'fails if importer raise an error' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_raise(Projects::ImportService::Error.new('Github: failed to connect API')) result = subject.execute diff --git a/spec/support/test_env.rb b/spec/support/test_env.rb index 71664bb192e..de5621314d1 100644 --- a/spec/support/test_env.rb +++ b/spec/support/test_env.rb @@ -78,8 +78,9 @@ module TestEnv end def setup_gitlab_shell - unless File.directory?(Rails.root.join(*%w(tmp tests gitlab-shell))) - `rake gitlab:shell:install` + unless File.directory?(Gitlab.config.gitlab_shell.path) + # TODO: Remove `[shards]` when gitlab-shell v3.1.0 is published + `rake gitlab:shell:install[shards]` end end @@ -125,14 +126,14 @@ module TestEnv def copy_repo(project) base_repo_path = File.expand_path(factory_repo_path_bare) - target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git") + target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git") FileUtils.mkdir_p(target_repo_path) FileUtils.cp_r("#{base_repo_path}/.", target_repo_path) FileUtils.chmod_R 0755, target_repo_path end def repos_path - Gitlab.config.gitlab_shell.repos_path + Gitlab.config.repositories.storages.default end def backup_path @@ -141,7 +142,7 @@ module TestEnv def copy_forked_repo_with_submodules(project) base_repo_path = File.expand_path(forked_repo_path_bare) - target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git") + target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git") FileUtils.mkdir_p(target_repo_path) FileUtils.cp_r("#{base_repo_path}/.", target_repo_path) FileUtils.chmod_R 0755, target_repo_path diff --git a/spec/workers/post_receive_spec.rb b/spec/workers/post_receive_spec.rb index b8e73682c91..20b1a343c27 100644 --- a/spec/workers/post_receive_spec.rb +++ b/spec/workers/post_receive_spec.rb @@ -91,6 +91,6 @@ describe PostReceive do end def pwd(project) - File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) + File.join(Gitlab.config.repositories.storages.default, project.path_with_namespace) end end diff --git a/spec/workers/repository_fork_worker_spec.rb b/spec/workers/repository_fork_worker_spec.rb index 4ef05eb29d2..5f762282b5e 100644 --- a/spec/workers/repository_fork_worker_spec.rb +++ b/spec/workers/repository_fork_worker_spec.rb @@ -14,6 +14,7 @@ describe RepositoryForkWorker do describe "#perform" do it "creates a new repository from a fork" do expect(shell).to receive(:fork_repository).with( + project.repository_storage_path, project.path_with_namespace, fork_project.namespace.path ).and_return(true) @@ -25,9 +26,11 @@ describe RepositoryForkWorker do end it 'flushes various caches' do - expect(shell).to receive(:fork_repository). - with(project.path_with_namespace, fork_project.namespace.path). - and_return(true) + expect(shell).to receive(:fork_repository).with( + project.repository_storage_path, + project.path_with_namespace, + fork_project.namespace.path + ).and_return(true) expect_any_instance_of(Repository).to receive(:expire_emptiness_caches). and_call_original |