1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
|
# frozen_string_literal: true
module Backup
# Backup and restores repositories using gitaly-backup
#
# gitaly-backup can work in parallel and accepts a list of repositories
# through input pipe using a specific json format for both backup and restore
class GitalyBackup
# @param [StringIO] progress IO interface to output progress
# @param [Integer] max_parallelism max parallelism when running backups
# @param [Integer] storage_parallelism max parallelism per storage (is affected by max_parallelism)
# @param [String] backup_id unique identifier for the backup
def initialize(progress, max_parallelism: nil, storage_parallelism: nil, incremental: false, backup_id: nil)
@progress = progress
@max_parallelism = max_parallelism
@storage_parallelism = storage_parallelism
@incremental = incremental
@backup_id = backup_id
end
def start(type, backup_repos_path)
raise Error, 'already started' if started?
command = case type
when :create
'create'
when :restore
'restore'
else
raise Error, "unknown backup type: #{type}"
end
args = []
args += ['-parallel', @max_parallelism.to_s] if @max_parallelism
args += ['-parallel-storage', @storage_parallelism.to_s] if @storage_parallelism
if Feature.enabled?(:incremental_repository_backup, default_enabled: :yaml)
args += ['-layout', 'pointer']
if type == :create
args += ['-incremental'] if @incremental
args += ['-id', @backup_id] if @backup_id
end
end
@input_stream, stdout, @thread = Open3.popen2(build_env, bin_path, command, '-path', backup_repos_path, *args)
@out_reader = Thread.new do
IO.copy_stream(stdout, @progress)
end
end
def finish!
return unless started?
@input_stream.close
[@thread, @out_reader].each(&:join)
status = @thread.value
@thread = nil
raise Error, "gitaly-backup exit status #{status.exitstatus}" if status.exitstatus != 0
end
def enqueue(container, repo_type)
raise Error, 'not started' unless started?
repository = repo_type.repository_for(container)
schedule_backup_job(repository, always_create: repo_type.project?)
end
def parallel_enqueue?
false
end
private
# Schedule a new backup job through a non-blocking JSON based pipe protocol
#
# @see https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/gitaly-backup.md
def schedule_backup_job(repository, always_create:)
connection_params = Gitlab::GitalyClient.connection_data(repository.storage)
json_job = {
address: connection_params['address'],
token: connection_params['token'],
storage_name: repository.storage,
relative_path: repository.relative_path,
gl_project_path: repository.gl_project_path,
always_create: always_create
}.to_json
@input_stream.puts(json_job)
end
def build_env
{
'SSL_CERT_FILE' => OpenSSL::X509::DEFAULT_CERT_FILE,
'SSL_CERT_DIR' => OpenSSL::X509::DEFAULT_CERT_DIR
}.merge(ENV)
end
def started?
@thread.present?
end
def bin_path
File.absolute_path(Gitlab.config.backup.gitaly_backup_path)
end
end
end
|
