1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
|
# frozen_string_literal: true
module Gitlab
module ImportExport
class RelationTreeRestorer
# Relations which cannot be saved at project level (and have a group assigned)
GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze
attr_reader :user
attr_reader :shared
attr_reader :importable
attr_reader :relation_reader
def initialize( # rubocop:disable Metrics/ParameterLists
user:, shared:, relation_reader:,
members_mapper:, object_builder:,
relation_factory:,
reader:,
importable:,
importable_attributes:,
importable_path:
)
@user = user
@shared = shared
@importable = importable
@relation_reader = relation_reader
@members_mapper = members_mapper
@object_builder = object_builder
@relation_factory = relation_factory
@reader = reader
@importable_attributes = importable_attributes
@importable_path = importable_path
end
def restore
ActiveRecord::Base.uncached do
ActiveRecord::Base.no_touching do
update_params!
BulkInsertableAssociations.with_bulk_insert(enabled: @importable.instance_of?(::Project)) do
fix_ci_pipelines_not_sorted_on_legacy_project_json!
create_relations!
end
end
end
# ensure that we have latest version of the restore
@importable.reload # rubocop:disable Cop/ActiveRecordAssociationReload
true
rescue StandardError => e
@shared.error(e)
false
end
private
# Loops through the tree of models defined in import_export.yml and
# finds them in the imported JSON so they can be instantiated and saved
# in the DB. The structure and relationships between models are guessed from
# the configuration yaml file too.
# Finally, it updates each attribute in the newly imported project/group.
def create_relations!
relations.each do |relation_key, relation_definition|
process_relation!(relation_key, relation_definition)
end
end
def process_relation!(relation_key, relation_definition)
@relation_reader.consume_relation(@importable_path, relation_key).each do |data_hash, relation_index|
process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
end
end
def process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
relation_object = build_relation(relation_key, relation_definition, relation_index, data_hash)
return unless relation_object
return if importable_class == ::Project && group_model?(relation_object)
relation_object.assign_attributes(importable_class_sym => @importable)
import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do
relation_object.save!
log_relation_creation(@importable, relation_key, relation_object)
end
rescue StandardError => e
import_failure_service.log_import_failure(
source: 'process_relation_item!',
relation_key: relation_key,
relation_index: relation_index,
exception: e)
end
def import_failure_service
@import_failure_service ||= ImportFailureService.new(@importable)
end
def relations
@relations ||=
@reader
.attributes_finder
.find_relations_tree(importable_class_sym)
.deep_stringify_keys
end
def update_params!
params = @importable_attributes.except(*relations.keys.map(&:to_s))
params = params.merge(present_override_params)
# Cleaning all imported and overridden params
params = Gitlab::ImportExport::AttributeCleaner.clean(
relation_hash: params,
relation_class: importable_class,
excluded_keys: excluded_keys_for_relation(importable_class_sym))
@importable.assign_attributes(params)
@importable.drop_visibility_level! if importable_class == ::Project
Gitlab::Timeless.timeless(@importable) do
@importable.save!
end
end
def present_override_params
# we filter out the empty strings from the overrides
# keeping the default values configured
override_params&.transform_values do |value|
value.is_a?(String) ? value.presence : value
end&.compact
end
def override_params
@importable_override_params ||= importable_override_params
end
def importable_override_params
if @importable.respond_to?(:import_data)
@importable.import_data&.data&.fetch('override_params', nil) || {}
else
{}
end
end
def build_relations(relation_key, relation_definition, relation_index, data_hashes)
data_hashes
.map { |data_hash| build_relation(relation_key, relation_definition, relation_index, data_hash) }
.tap { |entries| entries.compact! }
end
def build_relation(relation_key, relation_definition, relation_index, data_hash)
# TODO: This is hack to not create relation for the author
# Rather make `RelationFactory#set_note_author` to take care of that
return data_hash if relation_key == 'author' || already_restored?(data_hash)
# create relation objects recursively for all sub-objects
relation_definition.each do |sub_relation_key, sub_relation_definition|
transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index)
end
relation = @relation_factory.create(**relation_factory_params(relation_key, relation_index, data_hash))
if relation && !relation.valid?
@shared.logger.warn(
message: "[Project/Group Import] Invalid object relation built",
relation_key: relation_key,
relation_index: relation_index,
relation_class: relation.class.name,
error_messages: relation.errors.full_messages.join(". ")
)
end
relation
end
# Since we update the data hash in place as we restore relation items,
# and since we also de-duplicate items, we might encounter items that
# have already been restored in a previous iteration.
def already_restored?(relation_item)
!relation_item.is_a?(Hash)
end
def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index)
sub_data_hash = data_hash[sub_relation_key]
return unless sub_data_hash
# if object is a hash we can create simple object
# as it means that this is 1-to-1 vs 1-to-many
current_item =
if sub_data_hash.is_a?(Array)
build_relations(
sub_relation_key,
sub_relation_definition,
relation_index,
sub_data_hash).presence
else
build_relation(
sub_relation_key,
sub_relation_definition,
relation_index,
sub_data_hash)
end
if current_item
data_hash[sub_relation_key] = current_item
else
data_hash.delete(sub_relation_key)
end
end
def group_model?(relation_object)
GROUP_MODELS.include?(relation_object.class) && relation_object.group_id
end
def excluded_keys_for_relation(relation)
@reader.attributes_finder.find_excluded_keys(relation)
end
def importable_class
@importable.class
end
def importable_class_sym
importable_class.to_s.downcase.to_sym
end
def relation_factory_params(relation_key, relation_index, data_hash)
{
relation_index: relation_index,
relation_sym: relation_key.to_sym,
relation_hash: data_hash,
importable: @importable,
members_mapper: @members_mapper,
object_builder: @object_builder,
user: @user,
excluded_keys: excluded_keys_for_relation(relation_key)
}
end
# Temporary fix for https://gitlab.com/gitlab-org/gitlab/-/issues/27883 when import from legacy project.json
# This should be removed once legacy JSON format is deprecated.
# Ndjson export file will fix the order during project export.
def fix_ci_pipelines_not_sorted_on_legacy_project_json!
return unless relation_reader.legacy?
relation_reader.sort_ci_pipelines_by_id
end
# Enable logging of each top-level relation creation when Importing
# into a Group if feature flag is enabled
def log_relation_creation(importable, relation_key, relation_object)
root_ancestor_group = importable.try(:root_ancestor)
return unless root_ancestor_group
return unless root_ancestor_group.instance_of?(::Group)
return unless Feature.enabled?(:log_import_export_relation_creation, root_ancestor_group)
@shared.logger.info(
importable_type: importable.class.to_s,
importable_id: importable.id,
relation_key: relation_key,
relation_id: relation_object.id,
author_id: relation_object.try(:author_id),
message: '[Project/Group Import] Created new object relation'
)
end
end
end
end
|