summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--GITALY_SERVER_VERSION2
-rw-r--r--Gemfile2
-rw-r--r--Gemfile.lock4
-rw-r--r--Gemfile.rails5.lock4
-rw-r--r--changelogs/unreleased/use-backup-custom-hooks-gitaly.yml5
-rw-r--r--lib/backup/repository.rb130
-rw-r--r--lib/gitlab/gitaly_client/repository_service.rb42
7 files changed, 119 insertions, 70 deletions
diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION
index c64d9d48a48..a9a7f3fec01 100644
--- a/GITALY_SERVER_VERSION
+++ b/GITALY_SERVER_VERSION
@@ -1 +1 @@
-0.105.1
+0.107.0
diff --git a/Gemfile b/Gemfile
index 98622cdde84..4bdb255c498 100644
--- a/Gemfile
+++ b/Gemfile
@@ -419,7 +419,7 @@ group :ed25519 do
end
# Gitaly GRPC client
-gem 'gitaly-proto', '~> 0.101.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 0.102.0', require: 'gitaly'
gem 'grpc', '~> 1.11.0'
# Locked until https://github.com/google/protobuf/issues/4210 is closed
diff --git a/Gemfile.lock b/Gemfile.lock
index 883e580b86b..8d508cc0f39 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -283,7 +283,7 @@ GEM
gettext_i18n_rails (>= 0.7.1)
po_to_json (>= 1.0.0)
rails (>= 3.2.0)
- gitaly-proto (0.101.0)
+ gitaly-proto (0.102.0)
google-protobuf (~> 3.1)
grpc (~> 1.10)
github-linguist (5.3.3)
@@ -1039,7 +1039,7 @@ DEPENDENCIES
gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.3)
- gitaly-proto (~> 0.101.0)
+ gitaly-proto (~> 0.102.0)
github-linguist (~> 5.3.3)
gitlab-flowdock-git-hook (~> 1.0.1)
gitlab-gollum-lib (~> 4.2)
diff --git a/Gemfile.rails5.lock b/Gemfile.rails5.lock
index 223717f1818..fd560290d6c 100644
--- a/Gemfile.rails5.lock
+++ b/Gemfile.rails5.lock
@@ -286,7 +286,7 @@ GEM
gettext_i18n_rails (>= 0.7.1)
po_to_json (>= 1.0.0)
rails (>= 3.2.0)
- gitaly-proto (0.101.0)
+ gitaly-proto (0.102.0)
google-protobuf (~> 3.1)
grpc (~> 1.10)
github-linguist (5.3.3)
@@ -1049,7 +1049,7 @@ DEPENDENCIES
gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.3)
- gitaly-proto (~> 0.101.0)
+ gitaly-proto (~> 0.102.0)
github-linguist (~> 5.3.3)
gitlab-flowdock-git-hook (~> 1.0.1)
gitlab-gollum-lib (~> 4.2)
diff --git a/changelogs/unreleased/use-backup-custom-hooks-gitaly.yml b/changelogs/unreleased/use-backup-custom-hooks-gitaly.yml
new file mode 100644
index 00000000000..4b9766332c3
--- /dev/null
+++ b/changelogs/unreleased/use-backup-custom-hooks-gitaly.yml
@@ -0,0 +1,5 @@
+---
+title: migrate backup rake task to gitaly
+merge_request:
+author:
+type: added
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 0119c5d6851..221ba52b490 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -4,7 +4,6 @@ require_relative 'helper'
module Backup
class Repository
include Backup::Helper
- # rubocop:disable Metrics/AbcSize
attr_reader :progress
@@ -18,61 +17,26 @@ module Backup
Project.find_each(batch_size: 1000) do |project|
progress.print " * #{display_repo_path(project)} ... "
- path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
- path_to_repo(project)
- end
- path_to_project_bundle = path_to_bundle(project)
-
- # Create namespace dir or hashed path if missing
if project.hashed_storage?(:repository)
FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
else
FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
end
- if empty_repo?(project)
- progress.puts "[SKIPPED]".color(:cyan)
+ if !empty_repo?(project)
+ backup_project(project)
+ progress.puts "[DONE]".color(:green)
else
- in_path(path_to_project_repo) do |dir|
- FileUtils.mkdir_p(path_to_tars(project))
- cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
- output, status = Gitlab::Popen.popen(cmd)
-
- unless status.zero?
- progress_warn(project, cmd.join(' '), output)
- end
- end
-
- cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
- output, status = Gitlab::Popen.popen(cmd)
-
- if status.zero?
- progress.puts "[DONE]".color(:green)
- else
- progress_warn(project, cmd.join(' '), output)
- end
+ progress.puts "[SKIPPED]".color(:cyan)
end
wiki = ProjectWiki.new(project)
- path_to_wiki_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
- path_to_repo(wiki)
- end
- path_to_wiki_bundle = path_to_bundle(wiki)
- if File.exist?(path_to_wiki_repo)
- progress.print " * #{display_repo_path(wiki)} ... "
-
- if empty_repo?(wiki)
- progress.puts " [SKIPPED]".color(:cyan)
- else
- cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all)
- output, status = Gitlab::Popen.popen(cmd)
- if status.zero?
- progress.puts " [DONE]".color(:green)
- else
- progress_warn(wiki, cmd.join(' '), output)
- end
- end
+ if !empty_repo?(wiki)
+ backup_project(wiki)
+ progress.puts "[DONE] Wiki".color(:green)
+ else
+ progress.puts "[SKIPPED] Wiki".color(:cyan)
end
end
end
@@ -83,6 +47,38 @@ module Backup
end
end
+ def backup_project(project)
+ gitaly_migrate(:repository_backup) do |is_enabled|
+ if is_enabled
+ backup_project_gitaly(project)
+ else
+ backup_project_local(project)
+ end
+ end
+
+ backup_custom_hooks(project)
+ rescue => e
+ progress_warn(project, e, 'Failed to backup repo')
+ end
+
+ def backup_project_gitaly(project)
+ path_to_project_bundle = path_to_bundle(project)
+ Gitlab::GitalyClient::RepositoryService.new(project.repository)
+ .create_bundle(path_to_project_bundle)
+ end
+
+ def backup_project_local(project)
+ path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+ path_to_repo(project)
+ end
+
+ path_to_project_bundle = path_to_bundle(project)
+
+ cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
+ output, status = Gitlab::Popen.popen(cmd)
+ progress_warn(project, cmd.join(' '), output) unless status.zero?
+ end
+
def delete_all_repositories(name, repository_storage)
gitaly_migrate(:delete_all_repositories) do |is_enabled|
if is_enabled
@@ -97,8 +93,6 @@ module Backup
path = repository_storage.legacy_disk_path
return unless File.exist?(path)
- # Move all files in the existing repos directory except . and .. to
- # repositories.old.<timestamp> directory
bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
FileUtils.mkdir_p(bk_repos_path, mode: 0700)
files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
@@ -129,13 +123,47 @@ module Backup
.restore_custom_hooks(custom_hooks_path)
end
+ def local_backup_custom_hooks(project)
+ in_path(path_to_tars(project)) do |dir|
+ path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+ path_to_repo(project)
+ end
+ break unless File.exist?(File.join(path_to_project_repo, dir))
+
+ FileUtils.mkdir_p(path_to_tars(project))
+ cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir})
+ output, status = Gitlab::Popen.popen(cmd)
+
+ unless status.zero?
+ progress_warn(project, cmd.join(' '), output)
+ end
+ end
+ end
+
+ def gitaly_backup_custom_hooks(project)
+ FileUtils.mkdir_p(path_to_tars(project))
+ custom_hooks_path = path_to_tars(project, 'custom_hooks')
+ Gitlab::GitalyClient::RepositoryService.new(project.repository)
+ .backup_custom_hooks(custom_hooks_path)
+ end
+
+ def backup_custom_hooks(project)
+ gitaly_migrate(:backup_custom_hooks) do |is_enabled|
+ if is_enabled
+ gitaly_backup_custom_hooks(project)
+ else
+ local_backup_custom_hooks(project)
+ end
+ end
+ end
+
def restore_custom_hooks(project)
in_path(path_to_tars(project)) do |dir|
gitaly_migrate(:restore_custom_hooks) do |is_enabled|
if is_enabled
- local_restore_custom_hooks(project, dir)
- else
gitaly_restore_custom_hooks(project, dir)
+ else
+ local_restore_custom_hooks(project, dir)
end
end
end
@@ -186,7 +214,6 @@ module Backup
end
end
end
- # rubocop:enable Metrics/AbcSize
protected
@@ -224,9 +251,7 @@ module Backup
def prepare
FileUtils.rm_rf(backup_repos_path)
- # Ensure the parent dir of backup_repos_path exists
FileUtils.mkdir_p(Gitlab.config.backup.path)
- # Fail if somebody raced to create backup_repos_path before us
FileUtils.mkdir(backup_repos_path, mode: 0700)
end
@@ -242,7 +267,6 @@ module Backup
end
def empty_repo?(project_or_wiki)
- # Protect against stale caches
project_or_wiki.repository.expire_emptiness_caches
project_or_wiki.repository.empty?
end
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index 4340f779e53..ca986434221 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -196,20 +196,21 @@ module Gitlab
end
def create_bundle(save_path)
- request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo)
- response = GitalyClient.call(
- @storage,
- :repository_service,
+ gitaly_fetch_stream_to_file(
+ save_path,
:create_bundle,
- request,
- timeout: GitalyClient.default_timeout
+ Gitaly::CreateBundleRequest,
+ GitalyClient.default_timeout
)
+ end
- File.open(save_path, 'wb') do |f|
- response.each do |message|
- f.write(message.data)
- end
- end
+ def backup_custom_hooks(save_path)
+ gitaly_fetch_stream_to_file(
+ save_path,
+ :backup_custom_hooks,
+ Gitaly::BackupCustomHooksRequest,
+ GitalyClient.default_timeout
+ )
end
def create_from_bundle(bundle_path)
@@ -309,6 +310,25 @@ module Gitlab
private
+ def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout)
+ request = request_class.new(repository: @gitaly_repo)
+ response = GitalyClient.call(
+ @storage,
+ :repository_service,
+ rpc_name,
+ request,
+ timeout: timeout
+ )
+
+ File.open(save_path, 'wb') do |f|
+ response.each do |message|
+ f.write(message.data)
+ end
+ end
+ # If the file is empty means that we recieved an empty stream, we delete the file
+ FileUtils.rm(save_path) if File.zero?(save_path)
+ end
+
def gitaly_repo_stream_request(file_path, rpc_name, request_class, timeout)
request = request_class.new(repository: @gitaly_repo)
enum = Enumerator.new do |y|