summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlejandro Rodríguez <alejorro70@gmail.com>2016-06-14 12:27:33 -0400
committerAlejandro Rodríguez <alejorro70@gmail.com>2016-06-14 12:32:51 -0400
commit98c5e6295b201eeaa760d104f9b43e237f1dbd49 (patch)
tree9769f7903dabe8b5f8745ced9eb9f1bcf6d7fee4
parentc6cc7680e7da89c145b7e2075266791a3f594eec (diff)
downloadgitlab-ce-shards-test.tar.gz
Refactor repository paths handling to allow multiple git mount pointsshards-test
-rw-r--r--CHANGELOG1
-rw-r--r--GITLAB_SHELL_VERSION2
-rw-r--r--app/helpers/projects_helper.rb4
-rw-r--r--app/models/namespace.rb90
-rw-r--r--app/models/project.rb27
-rw-r--r--app/models/project_wiki.rb4
-rw-r--r--app/models/repository.rb2
-rw-r--r--app/services/projects/destroy_service.rb6
-rw-r--r--app/services/projects/housekeeping_service.rb2
-rw-r--r--app/services/projects/import_service.rb2
-rw-r--r--app/services/projects/transfer_service.rb4
-rw-r--r--app/views/projects/imports/new.html.haml2
-rw-r--r--app/workers/post_receive.rb6
-rw-r--r--app/workers/repository_fork_worker.rb2
-rw-r--r--config/gitlab.teatro.yml6
-rw-r--r--config/gitlab.yml.example13
-rw-r--r--config/initializers/1_settings.rb6
-rw-r--r--db/migrate/20160608195742_add_repository_storage_to_projects.rb5
-rw-r--r--db/schema.rb1
-rw-r--r--doc/raketasks/import.md3
-rw-r--r--lib/api/internal.rb36
-rw-r--r--lib/backup/repository.rb16
-rw-r--r--lib/gitlab/backend/shell.rb81
-rw-r--r--lib/gitlab/github_import/importer.rb2
-rw-r--r--lib/tasks/gitlab/check.rake165
-rw-r--r--lib/tasks/gitlab/cleanup.rake71
-rw-r--r--lib/tasks/gitlab/import.rake96
-rw-r--r--lib/tasks/gitlab/info.rake5
-rw-r--r--lib/tasks/gitlab/list_repos.rake2
-rw-r--r--lib/tasks/gitlab/shell.rake6
-rw-r--r--lib/tasks/gitlab/task_helpers.rake8
-rw-r--r--spec/helpers/projects_helper_spec.rb10
-rw-r--r--spec/lib/gitlab/backend/shell_spec.rb5
-rw-r--r--spec/models/namespace_spec.rb10
-rw-r--r--spec/models/project_spec.rb30
-rw-r--r--spec/requests/api/internal_spec.rb2
-rw-r--r--spec/services/destroy_group_service_spec.rb8
-rw-r--r--spec/services/projects/housekeeping_service_spec.rb2
-rw-r--r--spec/services/projects/import_service_spec.rb12
-rw-r--r--spec/support/test_env.rb11
-rw-r--r--spec/workers/post_receive_spec.rb2
-rw-r--r--spec/workers/repository_fork_worker_spec.rb9
42 files changed, 449 insertions, 328 deletions
diff --git a/CHANGELOG b/CHANGELOG
index 74fb52d3aeb..c054c11824d 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -79,6 +79,7 @@ v 8.9.0 (unreleased)
- Allow users to create confidential issues in private projects
- Measure CPU time for instrumented methods
- Instrument private methods and private instance methods by default instead just public methods
+ - Refactor repository paths handling to allow multiple git mount points
v 8.8.5 (unreleased)
- Ensure branch cleanup regardless of whether the GitHub import process succeeds
diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION
index 4a36342fcab..fd2a01863fd 100644
--- a/GITLAB_SHELL_VERSION
+++ b/GITLAB_SHELL_VERSION
@@ -1 +1 @@
-3.0.0
+3.1.0
diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb
index d30dd66202b..825dce30863 100644
--- a/app/helpers/projects_helper.rb
+++ b/app/helpers/projects_helper.rb
@@ -323,9 +323,9 @@ module ProjectsHelper
end
end
- def sanitize_repo_path(message)
+ def sanitize_repo_path(project, message)
return '' unless message.present?
- message.strip.gsub(Gitlab.config.gitlab_shell.repos_path.chomp('/'), "[REPOS PATH]")
+ message.strip.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]")
end
end
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index da19462f265..9c0594b1e06 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -21,8 +21,10 @@ class Namespace < ActiveRecord::Base
delegate :name, to: :owner, allow_nil: true, prefix: true
- after_create :ensure_dir_exist
after_update :move_dir, if: :path_changed?
+
+ # Save the storage paths before the projects are destroyed to use them on after destroy
+ before_destroy(prepend: true) { @old_repository_storage_paths = repository_storage_paths }
after_destroy :rm_dir
scope :root, -> { where('type IS NULL') }
@@ -87,51 +89,35 @@ class Namespace < ActiveRecord::Base
owner_name
end
- def ensure_dir_exist
- gitlab_shell.add_namespace(path)
- end
-
- def rm_dir
- # Move namespace directory into trash.
- # We will remove it later async
- new_path = "#{path}+#{id}+deleted"
-
- if gitlab_shell.mv_namespace(path, new_path)
- message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
- Gitlab::AppLogger.info message
-
- # Remove namespace directroy async with delay so
- # GitLab has time to remove all projects first
- GitlabShellWorker.perform_in(5.minutes, :rm_namespace, new_path)
- end
- end
-
def move_dir
- # Ensure old directory exists before moving it
- gitlab_shell.add_namespace(path_was)
-
if any_project_has_container_registry_tags?
raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry')
end
- if gitlab_shell.mv_namespace(path_was, path)
- Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
-
- # If repositories moved successfully we need to
- # send update instructions to users.
- # However we cannot allow rollback since we moved namespace dir
- # So we basically we mute exceptions in next actions
- begin
- send_update_instructions
- rescue
- # Returning false does not rollback after_* transaction but gives
- # us information about failing some of tasks
- false
+ # Move the namespace directory in all storages paths used by member projects
+ repository_storage_paths.each do |repository_storage_path|
+ # Ensure old directory exists before moving it
+ gitlab_shell.add_namespace(repository_storage_path, path_was)
+
+ unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
+ # if we cannot move namespace directory we should rollback
+ # db changes in order to prevent out of sync between db and fs
+ raise Exception.new('namespace directory cannot be moved')
end
- else
- # if we cannot move namespace directory we should rollback
- # db changes in order to prevent out of sync between db and fs
- raise Exception.new('namespace directory cannot be moved')
+ end
+
+ Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
+
+ # If repositories moved successfully we need to
+ # send update instructions to users.
+ # However we cannot allow rollback since we moved namespace dir
+ # So we basically we mute exceptions in next actions
+ begin
+ send_update_instructions
+ rescue
+ # Returning false does not rollback after_* transaction but gives
+ # us information about failing some of tasks
+ false
end
end
@@ -152,4 +138,28 @@ class Namespace < ActiveRecord::Base
def find_fork_of(project)
projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id)
end
+
+ private
+
+ def repository_storage_paths
+ projects.to_a.map(&:repository_storage_path).uniq
+ end
+
+ def rm_dir
+ # Remove the namespace directory in all storages paths used by member projects
+ @old_repository_storage_paths.each do |repository_storage_path|
+ # Move namespace directory into trash.
+ # We will remove it later async
+ new_path = "#{path}+#{id}+deleted"
+
+ if gitlab_shell.mv_namespace(repository_storage_path, path, new_path)
+ message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
+ Gitlab::AppLogger.info message
+
+ # Remove namespace directroy async with delay so
+ # GitLab has time to remove all projects first
+ GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
+ end
+ end
+ end
end
diff --git a/app/models/project.rb b/app/models/project.rb
index 0d2e612436a..4ea3b09c0c9 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -26,6 +26,15 @@ class Project < ActiveRecord::Base
default_value_for :container_registry_enabled, gitlab_config_features.container_registry
default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled }
+ # TODO: It used to be a namespace task to ensure the namespace directory
+ # existance, but since now projects might be in different paths (shards) they
+ # must account for that. However, checking the directory exists every time the
+ # project is saved might be inefficient. Consider moving this logic to another
+ # moment, or perhaps not to assume the directory exists and have gitlab-shell
+ # take that into consideration (doing `mkdir` himself before each operation,
+ # for example)
+ after_save :ensure_dir_exist
+
# set last_activity_at to the same as created_at
after_create :set_last_activity_at
def set_last_activity_at
@@ -351,6 +360,14 @@ class Project < ActiveRecord::Base
end
end
+ def repository_storage
+ super || 'default'
+ end
+
+ def repository_storage_path
+ Gitlab.config.repositories.storages[repository_storage]
+ end
+
def team
@team ||= ProjectTeam.new(self)
end
@@ -803,12 +820,12 @@ class Project < ActiveRecord::Base
raise Exception.new('Project cannot be renamed, because tags are present in its container registry')
end
- if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
+ if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
- gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
+ gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
send_move_instructions(old_path_with_namespace)
reset_events_cache
@@ -949,7 +966,7 @@ class Project < ActiveRecord::Base
def create_repository
# Forked import is handled asynchronously
unless forked?
- if gitlab_shell.add_repository(path_with_namespace)
+ if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
repository.after_create
true
else
@@ -1078,4 +1095,8 @@ class Project < ActiveRecord::Base
ensure
@errors = original_errors
end
+
+ def ensure_dir_exist
+ gitlab_shell.add_namespace(repository_storage_path, namespace.path)
+ end
end
diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb
index 25d82929c0b..a255710f577 100644
--- a/app/models/project_wiki.rb
+++ b/app/models/project_wiki.rb
@@ -159,7 +159,7 @@ class ProjectWiki
private
def init_repo(path_with_namespace)
- gitlab_shell.add_repository(path_with_namespace)
+ gitlab_shell.add_repository(project.repository_storage_path, path_with_namespace)
end
def commit_details(action, message = nil, title = nil)
@@ -173,7 +173,7 @@ class ProjectWiki
end
def path_to_repo
- @path_to_repo ||= File.join(Gitlab.config.gitlab_shell.repos_path, "#{path_with_namespace}.git")
+ @path_to_repo ||= File.join(project.repository_storage_path, "#{path_with_namespace}.git")
end
def update_project_activity
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 1ab163510bf..bb15be3a913 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -39,7 +39,7 @@ class Repository
# Return absolute path to repository
def path_to_repo
@path_to_repo ||= File.expand_path(
- File.join(Gitlab.config.gitlab_shell.repos_path, path_with_namespace + ".git")
+ File.join(@project.repository_storage_path, path_with_namespace + ".git")
)
end
diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb
index f09072975c3..882606e38d0 100644
--- a/app/services/projects/destroy_service.rb
+++ b/app/services/projects/destroy_service.rb
@@ -51,13 +51,13 @@ module Projects
return true if params[:skip_repo] == true
# There is a possibility project does not have repository or wiki
- return true unless gitlab_shell.exists?(path + '.git')
+ return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git')
new_path = removal_path(path)
- if gitlab_shell.mv_repository(path, new_path)
+ if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path)
log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
- GitlabShellWorker.perform_in(5.minutes, :remove_repository, new_path)
+ GitlabShellWorker.perform_in(5.minutes, :remove_repository, project.repository_storage_path, new_path)
else
false
end
diff --git a/app/services/projects/housekeeping_service.rb b/app/services/projects/housekeeping_service.rb
index 43db29315a1..a47df22f1ba 100644
--- a/app/services/projects/housekeeping_service.rb
+++ b/app/services/projects/housekeeping_service.rb
@@ -24,7 +24,7 @@ module Projects
def execute
raise LeaseTaken unless try_obtain_lease
- GitlabShellOneShotWorker.perform_async(:gc, @project.path_with_namespace)
+ GitlabShellOneShotWorker.perform_async(:gc, @project.repository_storage_path, @project.path_with_namespace)
ensure
Gitlab::Metrics.measure(:reset_pushes_since_gc) do
@project.update_column(:pushes_since_gc, 0)
diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb
index c4838d31f2f..70815b2dd5b 100644
--- a/app/services/projects/import_service.rb
+++ b/app/services/projects/import_service.rb
@@ -37,7 +37,7 @@ module Projects
def import_repository
begin
- gitlab_shell.import_repository(project.path_with_namespace, project.import_url)
+ gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url)
rescue Gitlab::Shell::Error => e
raise Error, "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
end
diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb
index 03b57dea51e..bc7f8bf433b 100644
--- a/app/services/projects/transfer_service.rb
+++ b/app/services/projects/transfer_service.rb
@@ -50,12 +50,12 @@ module Projects
project.send_move_instructions(old_path)
# Move main repository
- unless gitlab_shell.mv_repository(old_path, new_path)
+ unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, new_path)
raise TransferError.new('Cannot move project')
end
# Move wiki repo also if present
- gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki")
+ gitlab_shell.mv_repository(project.repository_storage_path, "#{old_path}.wiki", "#{new_path}.wiki")
# clear project cached events
project.reset_events_cache
diff --git a/app/views/projects/imports/new.html.haml b/app/views/projects/imports/new.html.haml
index a8a8caf7280..2cd8d03e30e 100644
--- a/app/views/projects/imports/new.html.haml
+++ b/app/views/projects/imports/new.html.haml
@@ -10,7 +10,7 @@
.panel-body
%pre
:preserve
- #{sanitize_repo_path(@project.import_error)}
+ #{sanitize_repo_path(@project, @project.import_error)}
= form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f|
= render "shared/import_form", f: f
diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb
index f3327ca9e61..09035a7cf2d 100644
--- a/app/workers/post_receive.rb
+++ b/app/workers/post_receive.rb
@@ -4,10 +4,10 @@ class PostReceive
sidekiq_options queue: :post_receive
def perform(repo_path, identifier, changes)
- if repo_path.start_with?(Gitlab.config.gitlab_shell.repos_path.to_s)
- repo_path.gsub!(Gitlab.config.gitlab_shell.repos_path.to_s, "")
+ if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) }
+ repo_path.gsub!(path[1].to_s, "")
else
- log("Check gitlab.yml config for correct gitlab_shell.repos_path variable. \"#{Gitlab.config.gitlab_shell.repos_path}\" does not match \"#{repo_path}\"")
+ log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"")
end
post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes)
diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb
index d947f105516..f7604e48f83 100644
--- a/app/workers/repository_fork_worker.rb
+++ b/app/workers/repository_fork_worker.rb
@@ -12,7 +12,7 @@ class RepositoryForkWorker
return
end
- result = gitlab_shell.fork_repository(source_path, target_path)
+ result = gitlab_shell.fork_repository(project.repository_storage_path, source_path, target_path)
unless result
logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}")
project.mark_import_as_failed('The project could not be forked.')
diff --git a/config/gitlab.teatro.yml b/config/gitlab.teatro.yml
index 01c8dc5ff98..75b79b837e0 100644
--- a/config/gitlab.teatro.yml
+++ b/config/gitlab.teatro.yml
@@ -47,11 +47,13 @@ production: &base
backup:
path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/)
+ repositories:
+ storages: # REPO PATHS MUST NOT BE A SYMLINK!!!
+ default: /apps/repositories/
+
gitlab_shell:
path: /apps/gitlab-shell/
- # REPOS_PATH MUST NOT BE A SYMLINK!!!
- repos_path: /apps/repositories/
hooks_path: /apps/gitlab-shell/hooks/
upload_pack: true
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 75e1a3c1093..7447f6a5cdd 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -428,6 +428,12 @@ production: &base
satellites:
path: /home/git/gitlab-satellites/
+ ## Repositories settings
+ repositories:
+ # Paths where repositories can be stored. Give the canonicalized absolute pathname.
+ # NOTE: REPOS PATHS MUST NOT CONTAIN ANY SYMLINK!!!
+ storages: # You must have at least a `default` storage path.
+
## Backup settings
backup:
path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/)
@@ -452,9 +458,6 @@ production: &base
## GitLab Shell settings
gitlab_shell:
path: /home/git/gitlab-shell/
-
- # REPOS_PATH MUST NOT BE A SYMLINK!!!
- repos_path: /home/git/repositories/
hooks_path: /home/git/gitlab-shell/hooks/
# File that contains the secret key for verifying access for gitlab-shell.
@@ -528,11 +531,13 @@ test:
# user: YOUR_USERNAME
satellites:
path: tmp/tests/gitlab-satellites/
+ repositories:
+ storages:
+ default: tmp/tests/repositories/
backup:
path: tmp/tests/backups
gitlab_shell:
path: tmp/tests/gitlab-shell/
- repos_path: tmp/tests/repositories/
hooks_path: tmp/tests/gitlab-shell/hooks/
issues_tracker:
redmine:
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 916fd33e767..dab657d1cc4 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -293,6 +293,11 @@ Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker'
#
+# Repositories
+#
+Settings.repositories.storages['default'] ||= Settings.gitlab['user_home'] + '/repositories/'
+
+#
# GitLab Shell
#
Settings['gitlab_shell'] ||= Settingslogic.new({})
@@ -301,7 +306,6 @@ Settings.gitlab_shell['hooks_path'] ||= Settings.gitlab['user_home'] + '/gitla
Settings.gitlab_shell['secret_file'] ||= Rails.root.join('.gitlab_shell_secret')
Settings.gitlab_shell['receive_pack'] = true if Settings.gitlab_shell['receive_pack'].nil?
Settings.gitlab_shell['upload_pack'] = true if Settings.gitlab_shell['upload_pack'].nil?
-Settings.gitlab_shell['repos_path'] ||= Settings.gitlab['user_home'] + '/repositories/'
Settings.gitlab_shell['ssh_host'] ||= Settings.gitlab.ssh_host
Settings.gitlab_shell['ssh_port'] ||= 22
Settings.gitlab_shell['ssh_user'] ||= Settings.gitlab.user
diff --git a/db/migrate/20160608195742_add_repository_storage_to_projects.rb b/db/migrate/20160608195742_add_repository_storage_to_projects.rb
new file mode 100644
index 00000000000..23d043e90a9
--- /dev/null
+++ b/db/migrate/20160608195742_add_repository_storage_to_projects.rb
@@ -0,0 +1,5 @@
+class AddRepositoryStorageToProjects < ActiveRecord::Migration
+ def change
+ add_column :projects, :repository_storage, :string
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 5fe39c5e59c..cd3ae15ab7f 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -779,6 +779,7 @@ ActiveRecord::Schema.define(version: 20160610301627) do
t.boolean "last_repository_check_failed"
t.datetime "last_repository_check_at"
t.boolean "container_registry_enabled"
+ t.string "repository_storage"
t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false
t.boolean "has_external_issue_tracker"
end
diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md
index 8a38937062e..2b305cb5c99 100644
--- a/doc/raketasks/import.md
+++ b/doc/raketasks/import.md
@@ -14,7 +14,8 @@
- For omnibus-gitlab, it is located at: `/var/opt/gitlab/git-data/repositories` by default, unless you changed
it in the `/etc/gitlab/gitlab.rb` file.
- For installations from source, it is usually located at: `/home/git/repositories` or you can see where
-your repositories are located by looking at `config/gitlab.yml` under the `gitlab_shell => repos_path` entry.
+your repositories are located by looking at `config/gitlab.yml` under the `repositories => storages` entries
+(you'll usually use the `default` storage path to start).
New folder needs to have git user ownership and read/write/execute access for git user and its group:
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 3ac7b50c4ce..de82783e3f3 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -20,6 +20,20 @@ module API
@wiki ||= params[:project].end_with?('.wiki') &&
!Project.find_with_namespace(params[:project])
end
+
+ def project
+ @project ||= begin
+ project_path = params[:project]
+
+ # Check for *.wiki repositories.
+ # Strip out the .wiki from the pathname before finding the
+ # project. This applies the correct project permissions to
+ # the wiki repository as well.
+ project_path.chomp!('.wiki') if wiki?
+
+ Project.find_with_namespace(project_path)
+ end
+ end
end
post "/allowed" do
@@ -34,16 +48,6 @@ module API
User.find_by(id: params[:user_id])
end
- project_path = params[:project]
-
- # Check for *.wiki repositories.
- # Strip out the .wiki from the pathname before finding the
- # project. This applies the correct project permissions to
- # the wiki repository as well.
- project_path.chomp!('.wiki') if wiki?
-
- project = Project.find_with_namespace(project_path)
-
access =
if wiki?
Gitlab::GitAccessWiki.new(actor, project)
@@ -51,7 +55,17 @@ module API
Gitlab::GitAccess.new(actor, project)
end
- access.check(params[:action], params[:changes])
+ access_status = access.check(params[:action], params[:changes])
+
+ response = { status: access_status.status, message: access_status.message }
+
+ if access_status.status
+ # Return the repository storage path so that gitlab-shell has it when
+ # handling ssh commands
+ response[:repository_storage_path] = project.repository_storage_path
+ end
+
+ response
end
#
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 7b91215d50b..9686b0b6f40 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -2,8 +2,6 @@ require 'yaml'
module Backup
class Repository
- attr_reader :repos_path
-
def dump
prepare
@@ -50,10 +48,12 @@ module Backup
end
def restore
- if File.exists?(repos_path)
+ Gitlab.config.repositories.storages.each do |name, path|
+ next unless File.exists?(path)
+
# Move repos dir to 'repositories.old' dir
- bk_repos_path = File.join(repos_path, '..', 'repositories.old.' + Time.now.to_i.to_s)
- FileUtils.mv(repos_path, bk_repos_path)
+ bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s)
+ FileUtils.mv(path, bk_repos_path)
end
FileUtils.mkdir_p(repos_path)
@@ -61,7 +61,7 @@ module Backup
Project.find_each(batch_size: 1000) do |project|
$progress.print " * #{project.path_with_namespace} ... "
- project.namespace.ensure_dir_exist if project.namespace
+ project.ensure_dir_exist
if File.exists?(path_to_bundle(project))
FileUtils.mkdir_p(path_to_repo(project))
@@ -120,10 +120,6 @@ module Backup
File.join(backup_repos_path, project.path_with_namespace + ".bundle")
end
- def repos_path
- Gitlab.config.gitlab_shell.repos_path
- end
-
def backup_repos_path
File.join(Gitlab.config.backup.path, "repositories")
end
diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb
index 3e3986d6382..e31840ef919 100644
--- a/lib/gitlab/backend/shell.rb
+++ b/lib/gitlab/backend/shell.rb
@@ -18,77 +18,82 @@ module Gitlab
# Init new repository
#
+ # storage - project's storage path
# name - project path with namespace
#
# Ex.
- # add_repository("gitlab/gitlab-ci")
+ # add_repository("/path/to/storage", "gitlab/gitlab-ci")
#
- def add_repository(name)
+ def add_repository(storage, name)
Gitlab::Utils.system_silent([gitlab_shell_projects_path,
- 'add-project', "#{name}.git"])
+ 'add-project', storage, "#{name}.git"])
end
# Import repository
#
+ # storage - project's storage path
# name - project path with namespace
#
# Ex.
- # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git")
+ # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git")
#
- def import_repository(name, url)
- output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900'])
+ def import_repository(storage, name, url)
+ output, status = Popen::popen([gitlab_shell_projects_path, 'import-project',
+ storage, "#{name}.git", url, '900'])
raise Error, output unless status.zero?
true
end
# Move repository
- #
+ # storage - project's storage path
# path - project path with namespace
# new_path - new project path with namespace
#
# Ex.
- # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new")
+ # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
#
- def mv_repository(path, new_path)
+ def mv_repository(storage, path, new_path)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project',
- "#{path}.git", "#{new_path}.git"])
+ storage, "#{path}.git", "#{new_path}.git"])
end
# Fork repository to new namespace
- #
+ # storage - project's storage path
# path - project path with namespace
# fork_namespace - namespace for forked project
#
# Ex.
- # fork_repository("gitlab/gitlab-ci", "randx")
+ # fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx")
#
- def fork_repository(path, fork_namespace)
+ def fork_repository(storage, path, fork_namespace)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project',
- "#{path}.git", fork_namespace])
+ storage, "#{path}.git", fork_namespace])
end
# Remove repository from file system
#
+ # storage - project's storage path
# name - project path with namespace
#
# Ex.
- # remove_repository("gitlab/gitlab-ci")
+ # remove_repository("/path/to/storage", "gitlab/gitlab-ci")
#
- def remove_repository(name)
+ def remove_repository(storage, name)
Gitlab::Utils.system_silent([gitlab_shell_projects_path,
- 'rm-project', "#{name}.git"])
+ 'rm-project', storage, "#{name}.git"])
end
# Gc repository
#
+ # storage - project storage path
# path - project path with namespace
#
# Ex.
- # gc("gitlab/gitlab-ci")
+ # gc("/path/to/storage", "gitlab/gitlab-ci")
#
- def gc(path)
+ def gc(storage, path)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc',
- "#{path}.git"])
+ storage, "#{path}.git"])
end
# Add new key to gitlab-shell
@@ -133,31 +138,31 @@ module Gitlab
# Add empty directory for storing repositories
#
# Ex.
- # add_namespace("gitlab")
+ # add_namespace("/path/to/storage", "gitlab")
#
- def add_namespace(name)
- FileUtils.mkdir(full_path(name), mode: 0770) unless exists?(name)
+ def add_namespace(storage, name)
+ FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name)
end
# Remove directory from repositories storage
# Every repository inside this directory will be removed too
#
# Ex.
- # rm_namespace("gitlab")
+ # rm_namespace("/path/to/storage", "gitlab")
#
- def rm_namespace(name)
- FileUtils.rm_r(full_path(name), force: true)
+ def rm_namespace(storage, name)
+ FileUtils.rm_r(full_path(storage, name), force: true)
end
# Move namespace directory inside repositories storage
#
# Ex.
- # mv_namespace("gitlab", "gitlabhq")
+ # mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
#
- def mv_namespace(old_name, new_name)
- return false if exists?(new_name) || !exists?(old_name)
+ def mv_namespace(storage, old_name, new_name)
+ return false if exists?(storage, new_name) || !exists?(storage, old_name)
- FileUtils.mv(full_path(old_name), full_path(new_name))
+ FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name))
end
def url_to_repo(path)
@@ -176,11 +181,11 @@ module Gitlab
# Check if such directory exists in repositories.
#
# Usage:
- # exists?('gitlab')
- # exists?('gitlab/cookies.git')
+ # exists?(storage, 'gitlab')
+ # exists?(storage, 'gitlab/cookies.git')
#
- def exists?(dir_name)
- File.exist?(full_path(dir_name))
+ def exists?(storage, dir_name)
+ File.exist?(full_path(storage, dir_name))
end
protected
@@ -193,14 +198,10 @@ module Gitlab
File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}")
end
- def repos_path
- Gitlab.config.gitlab_shell.repos_path
- end
-
- def full_path(dir_name)
+ def full_path(storage, dir_name)
raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?
- File.join(repos_path, dir_name)
+ File.join(storage, dir_name)
end
def gitlab_shell_projects_path
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index e5cf66a0371..85be1361a08 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -154,7 +154,7 @@ module Gitlab
def import_wiki
unless project.wiki_enabled?
wiki = WikiFormatter.new(project)
- gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url)
+ gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url)
project.update_attribute(:wiki_enabled, true)
end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 12d6ac45fb6..95a9d393ae7 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -356,97 +356,108 @@ namespace :gitlab do
########################
def check_repo_base_exists
- print "Repo base directory exists? ... "
+ puts "Repo base directory exists?"
- repo_base_path = Gitlab.config.gitlab_shell.repos_path
+ Gitlab.config.repositories.storages.each do |name, repo_base_path|
+ print "#{name}... "
- if File.exists?(repo_base_path)
- puts "yes".color(:green)
- else
- puts "no".color(:red)
- puts "#{repo_base_path} is missing".color(:red)
- try_fixing_it(
- "This should have been created when setting up GitLab Shell.",
- "Make sure it's set correctly in config/gitlab.yml",
- "Make sure GitLab Shell is installed correctly."
- )
- for_more_information(
- see_installation_guide_section "GitLab Shell"
- )
- fix_and_rerun
+ if File.exists?(repo_base_path)
+ puts "yes".color(:green)
+ else
+ puts "no".color(:red)
+ puts "#{repo_base_path} is missing".color(:red)
+ try_fixing_it(
+ "This should have been created when setting up GitLab Shell.",
+ "Make sure it's set correctly in config/gitlab.yml",
+ "Make sure GitLab Shell is installed correctly."
+ )
+ for_more_information(
+ see_installation_guide_section "GitLab Shell"
+ )
+ fix_and_rerun
+ end
end
end
def check_repo_base_is_not_symlink
- print "Repo base directory is a symlink? ... "
+ puts "Repo storage directories are symlinks?"
- repo_base_path = Gitlab.config.gitlab_shell.repos_path
- unless File.exists?(repo_base_path)
- puts "can't check because of previous errors".color(:magenta)
- return
- end
+ Gitlab.config.repositories.storages.each do |name, repo_base_path|
+ print "#{name}... "
- unless File.symlink?(repo_base_path)
- puts "no".color(:green)
- else
- puts "yes".color(:red)
- try_fixing_it(
- "Make sure it's set to the real directory in config/gitlab.yml"
- )
- fix_and_rerun
+ unless File.exists?(repo_base_path)
+ puts "can't check because of previous errors".color(:magenta)
+ return
+ end
+
+ unless File.symlink?(repo_base_path)
+ puts "no".color(:green)
+ else
+ puts "yes".color(:red)
+ try_fixing_it(
+ "Make sure it's set to the real directory in config/gitlab.yml"
+ )
+ fix_and_rerun
+ end
end
end
def check_repo_base_permissions
- print "Repo base access is drwxrws---? ... "
+ puts "Repo paths access is drwxrws---?"
- repo_base_path = Gitlab.config.gitlab_shell.repos_path
- unless File.exists?(repo_base_path)
- puts "can't check because of previous errors".color(:magenta)
- return
- end
+ Gitlab.config.repositories.storages.each do |name, repo_base_path|
+ print "#{name}... "
- if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770")
- puts "yes".color(:green)
- else
- puts "no".color(:red)
- try_fixing_it(
- "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}",
- "sudo chmod -R ug-s #{repo_base_path}",
- "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
- )
- for_more_information(
- see_installation_guide_section "GitLab Shell"
- )
- fix_and_rerun
+ unless File.exists?(repo_base_path)
+ puts "can't check because of previous errors".color(:magenta)
+ return
+ end
+
+ if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770")
+ puts "yes".color(:green)
+ else
+ puts "no".color(:red)
+ try_fixing_it(
+ "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}",
+ "sudo chmod -R ug-s #{repo_base_path}",
+ "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
+ )
+ for_more_information(
+ see_installation_guide_section "GitLab Shell"
+ )
+ fix_and_rerun
+ end
end
end
def check_repo_base_user_and_group
gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group
- print "Repo base owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}? ... "
+ puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?"
- repo_base_path = Gitlab.config.gitlab_shell.repos_path
- unless File.exists?(repo_base_path)
- puts "can't check because of previous errors".color(:magenta)
- return
- end
+ Gitlab.config.repositories.storages.each do |name, repo_base_path|
+ print "#{name}... "
- uid = uid_for(gitlab_shell_ssh_user)
- gid = gid_for(gitlab_shell_owner_group)
- if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid
- puts "yes".color(:green)
- else
- puts "no".color(:red)
- puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue)
- try_fixing_it(
- "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}"
- )
- for_more_information(
- see_installation_guide_section "GitLab Shell"
- )
- fix_and_rerun
+ unless File.exists?(repo_base_path)
+ puts "can't check because of previous errors".color(:magenta)
+ return
+ end
+
+ uid = uid_for(gitlab_shell_ssh_user)
+ gid = gid_for(gitlab_shell_owner_group)
+ if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid
+ puts "yes".color(:green)
+ else
+ puts "no".color(:red)
+ puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue)
+ try_fixing_it(
+ "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}"
+ )
+ for_more_information(
+ see_installation_guide_section "GitLab Shell"
+ )
+ fix_and_rerun
+ end
end
end
@@ -785,13 +796,13 @@ namespace :gitlab do
namespace :repo do
desc "GitLab | Check the integrity of the repositories managed by GitLab"
task check: :environment do
- namespace_dirs = Dir.glob(
- File.join(Gitlab.config.gitlab_shell.repos_path, '*')
- )
+ Gitlab.config.repositories.storages.each do |name, path|
+ namespace_dirs = Dir.glob(File.join(path, '*'))
- namespace_dirs.each do |namespace_dir|
- repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
- repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
+ namespace_dirs.each do |namespace_dir|
+ repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
+ repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
+ end
end
end
end
@@ -799,12 +810,12 @@ namespace :gitlab do
namespace :user do
desc "GitLab | Check the integrity of a specific user's repositories"
task :check_repos, [:username] => :environment do |t, args|
- username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue))
+ username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue))
user = User.find_by(username: username)
if user
repo_dirs = user.authorized_projects.map do |p|
File.join(
- Gitlab.config.gitlab_shell.repos_path,
+ p.repository_storage_path,
"#{p.path_with_namespace}.git"
)
end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index ab0028d6603..b7cbdc6cd78 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -5,36 +5,36 @@ namespace :gitlab do
warn_user_is_not_gitlab
remove_flag = ENV['REMOVE']
-
namespaces = Namespace.pluck(:path)
- git_base_path = Gitlab.config.gitlab_shell.repos_path
- all_dirs = Dir.glob(git_base_path + '/*')
+ Gitlab.config.repositories.storages.each do |name, git_base_path|
+ all_dirs = Dir.glob(git_base_path + '/*')
- puts git_base_path.color(:yellow)
- puts "Looking for directories to remove... "
+ puts git_base_path.color(:yellow)
+ puts "Looking for directories to remove... "
- all_dirs.reject! do |dir|
- # skip if git repo
- dir =~ /.git$/
- end
+ all_dirs.reject! do |dir|
+ # skip if git repo
+ dir =~ /.git$/
+ end
- all_dirs.reject! do |dir|
- dir_name = File.basename dir
+ all_dirs.reject! do |dir|
+ dir_name = File.basename dir
- # skip if namespace present
- namespaces.include?(dir_name)
- end
+ # skip if namespace present
+ namespaces.include?(dir_name)
+ end
- all_dirs.each do |dir_path|
+ all_dirs.each do |dir_path|
- if remove_flag
- if FileUtils.rm_rf dir_path
- puts "Removed...#{dir_path}".color(:red)
+ if remove_flag
+ if FileUtils.rm_rf dir_path
+ puts "Removed...#{dir_path}".color(:red)
+ else
+ puts "Cannot remove #{dir_path}".color(:red)
+ end
else
- puts "Cannot remove #{dir_path}".color(:red)
+ puts "Can be removed: #{dir_path}".color(:red)
end
- else
- puts "Can be removed: #{dir_path}".color(:red)
end
end
@@ -48,20 +48,21 @@ namespace :gitlab do
warn_user_is_not_gitlab
move_suffix = "+orphaned+#{Time.now.to_i}"
- repo_root = Gitlab.config.gitlab_shell.repos_path
- # Look for global repos (legacy, depth 1) and normal repos (depth 2)
- IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
- find.each_line do |path|
- path.chomp!
- repo_with_namespace = path.
- sub(repo_root, '').
- sub(%r{^/*}, '').
- chomp('.git').
- chomp('.wiki')
- next if Project.find_with_namespace(repo_with_namespace)
- new_path = path + move_suffix
- puts path.inspect + ' -> ' + new_path.inspect
- File.rename(path, new_path)
+ Gitlab.config.repositories.storages.each do |name, repo_root|
+ # Look for global repos (legacy, depth 1) and normal repos (depth 2)
+ IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
+ find.each_line do |path|
+ path.chomp!
+ repo_with_namespace = path.
+ sub(repo_root, '').
+ sub(%r{^/*}, '').
+ chomp('.git').
+ chomp('.wiki')
+ next if Project.find_with_namespace(repo_with_namespace)
+ new_path = path + move_suffix
+ puts path.inspect + ' -> ' + new_path.inspect
+ File.rename(path, new_path)
+ end
end
end
end
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
index 4753f00c26a..dbdd4e977e8 100644
--- a/lib/tasks/gitlab/import.rake
+++ b/lib/tasks/gitlab/import.rake
@@ -2,73 +2,73 @@ namespace :gitlab do
namespace :import do
# How to use:
#
- # 1. copy the bare repos under the repos_path (commonly /home/git/repositories)
+ # 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories)
# 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production
#
# Notes:
# * The project owner will set to the first administator of the system
# * Existing projects will be skipped
#
- desc "GitLab | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance"
+ desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
task repos: :environment do
+ Gitlab.config.repositories.storages.each do |name, git_base_path|
+ repos_to_import = Dir.glob(git_base_path + '/**/*.git')
- git_base_path = Gitlab.config.gitlab_shell.repos_path
- repos_to_import = Dir.glob(git_base_path + '/**/*.git')
+ repos_to_import.each do |repo_path|
+ # strip repo base path
+ repo_path[0..git_base_path.length] = ''
- repos_to_import.each do |repo_path|
- # strip repo base path
- repo_path[0..git_base_path.length] = ''
+ path = repo_path.sub(/\.git$/, '')
+ group_name, name = File.split(path)
+ group_name = nil if group_name == '.'
- path = repo_path.sub(/\.git$/, '')
- group_name, name = File.split(path)
- group_name = nil if group_name == '.'
+ puts "Processing #{repo_path}".color(:yellow)
- puts "Processing #{repo_path}".color(:yellow)
-
- if path.end_with?('.wiki')
- puts " * Skipping wiki repo"
- next
- end
+ if path.end_with?('.wiki')
+ puts " * Skipping wiki repo"
+ next
+ end
- project = Project.find_with_namespace(path)
+ project = Project.find_with_namespace(path)
- if project
- puts " * #{project.name} (#{repo_path}) exists"
- else
- user = User.admins.reorder("id").first
+ if project
+ puts " * #{project.name} (#{repo_path}) exists"
+ else
+ user = User.admins.reorder("id").first
- project_params = {
- name: name,
- path: name
- }
+ project_params = {
+ name: name,
+ path: name
+ }
- # find group namespace
- if group_name
- group = Namespace.find_by(path: group_name)
- # create group namespace
- unless group
- group = Group.new(:name => group_name)
- group.path = group_name
- group.owner = user
- if group.save
- puts " * Created Group #{group.name} (#{group.id})".color(:green)
- else
- puts " * Failed trying to create group #{group.name}".color(:red)
+ # find group namespace
+ if group_name
+ group = Namespace.find_by(path: group_name)
+ # create group namespace
+ unless group
+ group = Group.new(:name => group_name)
+ group.path = group_name
+ group.owner = user
+ if group.save
+ puts " * Created Group #{group.name} (#{group.id})".color(:green)
+ else
+ puts " * Failed trying to create group #{group.name}".color(:red)
+ end
end
+ # set project group
+ project_params[:namespace_id] = group.id
end
- # set project group
- project_params[:namespace_id] = group.id
- end
- project = Projects::CreateService.new(user, project_params).execute
+ project = Projects::CreateService.new(user, project_params).execute
- if project.persisted?
- puts " * Created #{project.name} (#{repo_path})".color(:green)
- project.update_repository_size
- project.update_commit_count
- else
- puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
- puts " Errors: #{project.errors.messages}".color(:red)
+ if project.persisted?
+ puts " * Created #{project.name} (#{repo_path})".color(:green)
+ project.update_repository_size
+ project.update_commit_count
+ else
+ puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
+ puts " Errors: #{project.errors.messages}".color(:red)
+ end
end
end
end
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index 352b566df24..fe43d40e6d2 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -62,7 +62,10 @@ namespace :gitlab do
puts ""
puts "GitLab Shell".color(:yellow)
puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}"
- puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}"
+ puts "Repository storage paths:"
+ Gitlab.config.repositories.storages.each do |name, path|
+ puts "- #{name}: \t#{path}"
+ end
puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}"
puts "Git:\t\t#{Gitlab.config.git.bin_path}"
diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake
index c7596e7abcb..ffcc76e5498 100644
--- a/lib/tasks/gitlab/list_repos.rake
+++ b/lib/tasks/gitlab/list_repos.rake
@@ -9,7 +9,7 @@ namespace :gitlab do
scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids)
end
scope.find_each do |project|
- base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace)
+ base = File.join(project.repository_storage_path, project.path_with_namespace)
puts base + '.git'
puts base + '.wiki.git'
end
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index b1648a4602a..00f0234923a 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -12,7 +12,6 @@ namespace :gitlab do
gitlab_url = Gitlab.config.gitlab.url
# gitlab-shell requires a / at the end of the url
gitlab_url += '/' unless gitlab_url.end_with?('/')
- repos_path = Gitlab.config.gitlab_shell.repos_path
target_dir = Gitlab.config.gitlab_shell.path
# Clone if needed
@@ -35,7 +34,6 @@ namespace :gitlab do
user: user,
gitlab_url: gitlab_url,
http_settings: {self_signed_cert: false}.stringify_keys,
- repos_path: repos_path,
auth_file: File.join(home_dir, ".ssh", "authorized_keys"),
redis: {
bin: %x{which redis-cli}.chomp,
@@ -87,7 +85,8 @@ namespace :gitlab do
if File.exists?(path_to_repo)
print '-'
else
- if Gitlab::Shell.new.add_repository(project.path_with_namespace)
+ if Gitlab::Shell.new.add_repository(project.repository_storage_path,
+ project.path_with_namespace)
print '.'
else
print 'F'
@@ -138,4 +137,3 @@ namespace :gitlab do
system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag}))
end
end
-
diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake
index d0c019044b7..68a099f8307 100644
--- a/lib/tasks/gitlab/task_helpers.rake
+++ b/lib/tasks/gitlab/task_helpers.rake
@@ -125,9 +125,11 @@ namespace :gitlab do
end
def all_repos
- IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
- find.each_line do |path|
- yield path.chomp
+ Gitlab.config.repositories.storages.each do |name, path|
+ IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
+ find.each_line do |path|
+ yield path.chomp
+ end
end
end
end
diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb
index 09e0bbfd00b..604204cca0a 100644
--- a/spec/helpers/projects_helper_spec.rb
+++ b/spec/helpers/projects_helper_spec.rb
@@ -123,11 +123,17 @@ describe ProjectsHelper do
end
describe '#sanitized_import_error' do
+ let(:project) { create(:project) }
+
+ before do
+ allow(project).to receive(:repository_storage_path).and_return('/base/repo/path')
+ end
+
it 'removes the repo path' do
- repo = File.join(Gitlab.config.gitlab_shell.repos_path, '/namespace/test.git')
+ repo = '/base/repo/path/namespace/test.git'
import_error = "Could not clone #{repo}\n"
- expect(sanitize_repo_path(import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git')
+ expect(sanitize_repo_path(project, import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git')
end
end
end
diff --git a/spec/lib/gitlab/backend/shell_spec.rb b/spec/lib/gitlab/backend/shell_spec.rb
index fd869f48b5c..e15f13f985b 100644
--- a/spec/lib/gitlab/backend/shell_spec.rb
+++ b/spec/lib/gitlab/backend/shell_spec.rb
@@ -13,6 +13,11 @@ describe Gitlab::Shell, lib: true do
it { is_expected.to respond_to :add_repository }
it { is_expected.to respond_to :remove_repository }
it { is_expected.to respond_to :fork_repository }
+ it { is_expected.to respond_to :gc }
+ it { is_expected.to respond_to :add_namespace }
+ it { is_expected.to respond_to :rm_namespace }
+ it { is_expected.to respond_to :mv_namespace }
+ it { is_expected.to respond_to :exists? }
it { expect(gitlab_shell.url_to_repo('diaspora')).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + "diaspora.git") }
diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb
index 4e68ac5e63a..cbea407f9bf 100644
--- a/spec/models/namespace_spec.rb
+++ b/spec/models/namespace_spec.rb
@@ -57,6 +57,7 @@ describe Namespace, models: true do
describe :move_dir do
before do
@namespace = create :namespace
+ @project = create :project, namespace: @namespace
allow(@namespace).to receive(:path_changed?).and_return(true)
end
@@ -87,8 +88,13 @@ describe Namespace, models: true do
end
describe :rm_dir do
- it "should remove dir" do
- expect(namespace.rm_dir).to be_truthy
+ let!(:project) { create(:project, namespace: namespace) }
+ let!(:path) { File.join(Gitlab.config.repositories.storages.default, namespace.path) }
+
+ before { namespace.destroy }
+
+ it "should remove its dirs when deleted" do
+ expect(File.exist?(path)).to be(false)
end
end
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index 30aa2b70c8d..7b44c7160b8 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -108,6 +108,28 @@ describe Project, models: true do
end
end
+ describe '#repository_storage' do
+ let(:project) { create(:project, repository_storage: nil) }
+
+ it 'should return the default repository storage if nil' do
+ expect(project.repository_storage).to eq('default')
+ end
+ end
+
+ describe '#repository_storage_path' do
+ let(:project) { create(:project, repository_storage: 'custom') }
+
+ before do
+ FileUtils.mkdir('tmp/tests/custom_repositories')
+ storages = { 'custom' => 'tmp/tests/custom_repositories' }
+ allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
+ end
+
+ it 'returns the repository storage path' do
+ expect(project.repository_storage_path).to eq('tmp/tests/custom_repositories')
+ end
+ end
+
it 'should return valid url to repo' do
project = Project.new(path: 'somewhere')
expect(project.url_to_repo).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + 'somewhere.git')
@@ -690,12 +712,12 @@ describe Project, models: true do
expect(gitlab_shell).to receive(:mv_repository).
ordered.
- with("#{ns}/foo", "#{ns}/#{project.path}").
+ with(project.repository_storage_path, "#{ns}/foo", "#{ns}/#{project.path}").
and_return(true)
expect(gitlab_shell).to receive(:mv_repository).
ordered.
- with("#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki").
+ with(project.repository_storage_path, "#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki").
and_return(true)
expect_any_instance_of(SystemHooksService).
@@ -787,7 +809,7 @@ describe Project, models: true do
context 'using a regular repository' do
it 'creates the repository' do
expect(shell).to receive(:add_repository).
- with(project.path_with_namespace).
+ with(project.repository_storage_path, project.path_with_namespace).
and_return(true)
expect(project.repository).to receive(:after_create)
@@ -797,7 +819,7 @@ describe Project, models: true do
it 'adds an error if the repository could not be created' do
expect(shell).to receive(:add_repository).
- with(project.path_with_namespace).
+ with(project.repository_storage_path, project.path_with_namespace).
and_return(false)
expect(project.repository).not_to receive(:after_create)
diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb
index 22802dd0e05..097398442f4 100644
--- a/spec/requests/api/internal_spec.rb
+++ b/spec/requests/api/internal_spec.rb
@@ -72,6 +72,7 @@ describe API::API, api: true do
expect(response.status).to eq(200)
expect(json_response["status"]).to be_truthy
+ expect(json_response["repository_storage_path"]).to eq(project.repository_storage_path)
end
end
@@ -81,6 +82,7 @@ describe API::API, api: true do
expect(response.status).to eq(200)
expect(json_response["status"]).to be_truthy
+ expect(json_response["repository_storage_path"]).to eq(project.repository_storage_path)
end
end
end
diff --git a/spec/services/destroy_group_service_spec.rb b/spec/services/destroy_group_service_spec.rb
index afa89b84175..eca8ddd8ea4 100644
--- a/spec/services/destroy_group_service_spec.rb
+++ b/spec/services/destroy_group_service_spec.rb
@@ -23,8 +23,8 @@ describe DestroyGroupService, services: true do
Sidekiq::Testing.inline! { destroy_group(group, user) }
end
- it { expect(gitlab_shell.exists?(group.path)).to be_falsey }
- it { expect(gitlab_shell.exists?(remove_path)).to be_falsey }
+ it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey }
+ it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_falsey }
end
context 'Sidekiq fake' do
@@ -33,8 +33,8 @@ describe DestroyGroupService, services: true do
Sidekiq::Testing.fake! { destroy_group(group, user) }
end
- it { expect(gitlab_shell.exists?(group.path)).to be_falsey }
- it { expect(gitlab_shell.exists?(remove_path)).to be_truthy }
+ it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey }
+ it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_truthy }
end
end
diff --git a/spec/services/projects/housekeeping_service_spec.rb b/spec/services/projects/housekeeping_service_spec.rb
index 4c5ced7e746..bd4dc6a0f79 100644
--- a/spec/services/projects/housekeeping_service_spec.rb
+++ b/spec/services/projects/housekeeping_service_spec.rb
@@ -12,7 +12,7 @@ describe Projects::HousekeepingService do
it 'enqueues a sidekiq job' do
expect(subject).to receive(:try_obtain_lease).and_return(true)
- expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.path_with_namespace)
+ expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.repository_storage_path, project.path_with_namespace)
subject.execute
expect(project.pushes_since_gc).to eq(0)
diff --git a/spec/services/projects/import_service_spec.rb b/spec/services/projects/import_service_spec.rb
index 068c9a1219c..d5d4d7c56ef 100644
--- a/spec/services/projects/import_service_spec.rb
+++ b/spec/services/projects/import_service_spec.rb
@@ -36,7 +36,7 @@ describe Projects::ImportService, services: true do
end
it 'succeeds if repository import is successfully' do
- expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true)
+ expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
result = subject.execute
@@ -44,7 +44,7 @@ describe Projects::ImportService, services: true do
end
it 'fails if repository import fails' do
- expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository'))
+ expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository'))
result = subject.execute
@@ -64,7 +64,7 @@ describe Projects::ImportService, services: true do
end
it 'succeeds if importer succeeds' do
- expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true)
+ expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(true)
result = subject.execute
@@ -74,7 +74,7 @@ describe Projects::ImportService, services: true do
it 'flushes various caches' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).
- with(project.path_with_namespace, project.import_url).
+ with(project.repository_storage_path, project.path_with_namespace, project.import_url).
and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).
@@ -90,7 +90,7 @@ describe Projects::ImportService, services: true do
end
it 'fails if importer fails' do
- expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true)
+ expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(false)
result = subject.execute
@@ -100,7 +100,7 @@ describe Projects::ImportService, services: true do
end
it 'fails if importer raise an error' do
- expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true)
+ expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_raise(Projects::ImportService::Error.new('Github: failed to connect API'))
result = subject.execute
diff --git a/spec/support/test_env.rb b/spec/support/test_env.rb
index 71664bb192e..de5621314d1 100644
--- a/spec/support/test_env.rb
+++ b/spec/support/test_env.rb
@@ -78,8 +78,9 @@ module TestEnv
end
def setup_gitlab_shell
- unless File.directory?(Rails.root.join(*%w(tmp tests gitlab-shell)))
- `rake gitlab:shell:install`
+ unless File.directory?(Gitlab.config.gitlab_shell.path)
+ # TODO: Remove `[shards]` when gitlab-shell v3.1.0 is published
+ `rake gitlab:shell:install[shards]`
end
end
@@ -125,14 +126,14 @@ module TestEnv
def copy_repo(project)
base_repo_path = File.expand_path(factory_repo_path_bare)
- target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git")
+ target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git")
FileUtils.mkdir_p(target_repo_path)
FileUtils.cp_r("#{base_repo_path}/.", target_repo_path)
FileUtils.chmod_R 0755, target_repo_path
end
def repos_path
- Gitlab.config.gitlab_shell.repos_path
+ Gitlab.config.repositories.storages.default
end
def backup_path
@@ -141,7 +142,7 @@ module TestEnv
def copy_forked_repo_with_submodules(project)
base_repo_path = File.expand_path(forked_repo_path_bare)
- target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git")
+ target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git")
FileUtils.mkdir_p(target_repo_path)
FileUtils.cp_r("#{base_repo_path}/.", target_repo_path)
FileUtils.chmod_R 0755, target_repo_path
diff --git a/spec/workers/post_receive_spec.rb b/spec/workers/post_receive_spec.rb
index b8e73682c91..20b1a343c27 100644
--- a/spec/workers/post_receive_spec.rb
+++ b/spec/workers/post_receive_spec.rb
@@ -91,6 +91,6 @@ describe PostReceive do
end
def pwd(project)
- File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace)
+ File.join(Gitlab.config.repositories.storages.default, project.path_with_namespace)
end
end
diff --git a/spec/workers/repository_fork_worker_spec.rb b/spec/workers/repository_fork_worker_spec.rb
index 4ef05eb29d2..5f762282b5e 100644
--- a/spec/workers/repository_fork_worker_spec.rb
+++ b/spec/workers/repository_fork_worker_spec.rb
@@ -14,6 +14,7 @@ describe RepositoryForkWorker do
describe "#perform" do
it "creates a new repository from a fork" do
expect(shell).to receive(:fork_repository).with(
+ project.repository_storage_path,
project.path_with_namespace,
fork_project.namespace.path
).and_return(true)
@@ -25,9 +26,11 @@ describe RepositoryForkWorker do
end
it 'flushes various caches' do
- expect(shell).to receive(:fork_repository).
- with(project.path_with_namespace, fork_project.namespace.path).
- and_return(true)
+ expect(shell).to receive(:fork_repository).with(
+ project.repository_storage_path,
+ project.path_with_namespace,
+ fork_project.namespace.path
+ ).and_return(true)
expect_any_instance_of(Repository).to receive(:expire_emptiness_caches).
and_call_original