1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
# frozen_string_literal: true
module Gitlab
module ImportExport
module Json
class LegacyReader
class File < LegacyReader
include Gitlab::Utils::StrongMemoize
def initialize(path, relation_names:, allowed_path: nil)
@path = path
super(
relation_names: relation_names,
allowed_path: allowed_path)
end
def exist?
::File.exist?(@path)
end
protected
def tree_hash
strong_memoize(:tree_hash) do
read_hash
end
end
def read_hash
ActiveSupport::JSON.decode(IO.read(@path))
rescue StandardError => e
Gitlab::ErrorTracking.log_exception(e)
raise Gitlab::ImportExport::Error, 'Incorrect JSON format'
end
end
class Hash < LegacyReader
def initialize(tree_hash, relation_names:, allowed_path: nil)
@tree_hash = tree_hash
super(
relation_names: relation_names,
allowed_path: allowed_path)
end
def exist?
@tree_hash.present?
end
protected
attr_reader :tree_hash
end
def initialize(relation_names:, allowed_path:)
@relation_names = relation_names.map(&:to_s)
@consumed_relations = Set.new
# This is legacy reader, to be used in transition
# period before `.ndjson`,
# we strong validate what is being readed
@allowed_path = allowed_path
end
def exist?
raise NotImplementedError
end
def legacy?
true
end
def consume_attributes(importable_path)
unless importable_path == @allowed_path
raise ArgumentError, "Invalid #{importable_path} passed to `consume_attributes`. Use #{@allowed_path} instead."
end
attributes
end
def consume_relation(importable_path, key)
unless importable_path == @allowed_path
raise ArgumentError, "Invalid #{importable_name} passed to `consume_relation`. Use #{@allowed_path} instead."
end
Enumerator.new do |documents|
next unless @consumed_relations.add?("#{importable_path}/#{key}")
value = relations.delete(key)
next if value.nil?
if value.is_a?(Array)
value.each.with_index do |item, idx|
documents << [item, idx]
end
else
documents << [value, 0]
end
end
end
def sort_ci_pipelines_by_id
relations['ci_pipelines']&.sort_by! { |hash| hash['id'] }
end
private
attr_reader :relation_names, :allowed_path
def tree_hash
raise NotImplementedError
end
def attributes
@attributes ||= tree_hash.slice!(*relation_names)
end
def relations
@relations ||= tree_hash.extract!(*relation_names)
end
end
end
end
end
|
