1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
|
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# Class that will fill the project_repositories table for projects that
# are on hashed storage and an entry is missing in this table.
class BackfillSnippetRepositories
MAX_RETRIES = 2
def perform(start_id, stop_id)
snippets = snippet_relation.where(id: start_id..stop_id)
migrate_snippets(snippets)
end
def perform_by_ids(snippet_ids)
snippets = snippet_relation.where(id: snippet_ids)
migrate_snippets(snippets)
end
private
def migrate_snippets(snippets)
snippets.find_each do |snippet|
# We need to expire the exists? value for the cached method in case it was cached
snippet.repository.expire_exists_cache
next if repository_present?(snippet)
retry_index = 0
@invalid_path_error = false
@invalid_signature_error = false
begin
create_repository_and_files(snippet)
logger.info(message: 'Snippet Migration: repository created and migrated', snippet: snippet.id)
rescue StandardError => e
set_file_path_error(e)
set_signature_error(e)
retry_index += 1
retry if retry_index < max_retries
logger.error(message: "Snippet Migration: error migrating snippet. Reason: #{e.message}", snippet: snippet.id)
destroy_snippet_repository(snippet)
delete_repository(snippet)
end
end
end
def snippet_relation
@snippet_relation ||= Snippet.includes(:author, snippet_repository: :shard)
end
def repository_present?(snippet)
snippet.snippet_repository && !snippet.empty_repo?
end
def create_repository_and_files(snippet)
snippet.create_repository
create_commit(snippet)
end
# Removing the db record
def destroy_snippet_repository(snippet)
snippet.snippet_repository&.delete
rescue StandardError => e
logger.error(message: "Snippet Migration: error destroying snippet repository. Reason: #{e.message}", snippet: snippet.id)
end
# Removing the repository in disk
def delete_repository(snippet)
return unless snippet.repository_exists?
snippet.repository.remove
snippet.repository.expire_exists_cache
rescue StandardError => e
logger.error(message: "Snippet Migration: error deleting repository. Reason: #{e.message}", snippet: snippet.id)
end
def logger
@logger ||= Gitlab::BackgroundMigration::Logger.build
end
def snippet_action(snippet)
# We don't need the previous_path param
# Because we're not updating any existing file
[{ file_path: filename(snippet),
content: snippet.content }]
end
def filename(snippet)
file_name = snippet.file_name
file_name = file_name.parameterize if @invalid_path_error
file_name.presence || empty_file_name
end
def empty_file_name
@empty_file_name ||= "#{SnippetRepository::DEFAULT_EMPTY_FILE_NAME}1.txt"
end
def commit_attrs
@commit_attrs ||= { branch_name: 'master', message: 'Initial commit' }
end
def create_commit(snippet)
snippet.snippet_repository.multi_files_action(commit_author(snippet), snippet_action(snippet), **commit_attrs)
end
# If the user is not allowed to access git or update the snippet
# because it is blocked, internal, ghost, ... we cannot commit
# files because these users are not allowed to, but we need to
# migrate their snippets as well.
# In this scenario the migration bot user will be the one that will commit the files.
def commit_author(snippet)
return migration_bot_user if snippet_content_size_over_limit?(snippet)
return migration_bot_user if @invalid_signature_error
if Gitlab::UserAccessSnippet.new(snippet.author, snippet: snippet).can_do_action?(:update_snippet)
snippet.author
else
migration_bot_user
end
end
def migration_bot_user
@migration_bot_user ||= User.migration_bot
end
# We sometimes receive invalid path errors from Gitaly if the Snippet filename
# cannot be parsed into a valid git path.
# In this situation, we need to parameterize the file name of the Snippet so that
# the migration can succeed, to achieve that, we'll identify in migration retries
# that the path is invalid
def set_file_path_error(error)
@invalid_path_error ||= error.is_a?(SnippetRepository::InvalidPathError)
end
# We sometimes receive invalid signature from Gitaly if the commit author
# name or email is invalid to create the commit signature.
# In this situation, we set the error and use the migration_bot since
# the information used to build it is valid
def set_signature_error(error)
@invalid_signature_error ||= error.is_a?(SnippetRepository::InvalidSignatureError)
end
# In the case where the snippet file_name is invalid and also the
# snippet author has invalid commit info, we need to increase the
# number of retries by 1, because we will receive two errors
# from Gitaly and, in the third one, we will commit successfully.
def max_retries
MAX_RETRIES + (@invalid_signature_error && @invalid_path_error ? 1 : 0)
end
def snippet_content_size_over_limit?(snippet)
snippet.content.size > Gitlab::CurrentSettings.snippet_size_limit
end
end
end
end
|
