1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
|
# frozen_string_literal: true
require 'spec_helper'
# This list is used to provide temporary exceptions for feature categories
# that are transitioning and not yet in the feature_categories.yml file
# any additions here should be accompanied by a link to an issue link
VALID_FEATURE_CATEGORIES = [
'jihu' # https://gitlab.com/gitlab-org/database-team/team-tasks/-/issues/192
].freeze
RSpec.shared_examples 'validate dictionary' do |objects, directory_path, required_fields|
context 'for each object' do
let(:directory_path) { directory_path }
let(:metadata_allowed_fields) do
required_fields + %i[
feature_categories
classes
description
introduced_by_url
milestone
gitlab_schema
]
end
let(:metadata) do
objects.each_with_object({}) do |object_name, hash|
next unless File.exist?(object_metadata_file_path(object_name))
hash[object_name] ||= load_object_metadata(required_fields, object_name)
end
end
let(:objects_without_metadata) do
objects.reject { |t| metadata.has_key?(t) }
end
let(:objects_without_valid_metadata) do
metadata.select { |_, t| t.has_key?(:error) }.keys
end
let(:objects_with_disallowed_fields) do
metadata.select { |_, t| t.has_key?(:disallowed_fields) }.keys
end
let(:objects_with_missing_required_fields) do
metadata.select { |_, t| t.has_key?(:missing_required_fields) }.keys
end
let(:objects_with_invalid_feature_category) do
metadata.select { |_, t| t.has_key?(:invalid_feature_category) }.keys
end
it 'has a metadata file' do
expect(objects_without_metadata).to be_empty, multiline_error(
'Missing metadata files',
objects_without_metadata.map { |t| " #{object_metadata_file(t)}" }
)
end
it 'has a valid metadata file' do
expect(objects_without_valid_metadata).to be_empty, object_metadata_errors(
'Table metadata files with errors',
:error,
objects_without_valid_metadata
)
end
it 'has a valid feature category' do
expect(objects_with_invalid_feature_category).to be_empty, object_metadata_errors(
'Table metadata files with an invalid feature category',
:error,
objects_with_invalid_feature_category
)
end
it 'has a valid metadata file with allowed fields' do
expect(objects_with_disallowed_fields).to be_empty, object_metadata_errors(
'Table metadata files with disallowed fields',
:disallowed_fields,
objects_with_disallowed_fields
)
end
it 'has a valid metadata file without missing fields' do
expect(objects_with_missing_required_fields).to be_empty, object_metadata_errors(
'Table metadata files with missing fields',
:missing_required_fields,
objects_with_missing_required_fields
)
end
end
private
def object_metadata_file(object_name)
File.join(directory_path, "#{object_name}.yml")
end
def object_metadata_file_path(object_name)
Rails.root.join(object_metadata_file(object_name))
end
def feature_categories_valid?(object_feature_categories)
return false unless object_feature_categories.present?
all_feature_categories = YAML.load_file(Rails.root.join('config/feature_categories.yml')) + VALID_FEATURE_CATEGORIES
object_feature_categories.all? { |category| all_feature_categories.include?(category) }
end
def load_object_metadata(required_fields, object_name)
result = {}
begin
result[:metadata] = YAML.safe_load(File.read(object_metadata_file_path(object_name))).deep_symbolize_keys
disallowed_fields = (result[:metadata].keys - metadata_allowed_fields)
result[:disallowed_fields] = "fields not allowed: #{disallowed_fields.join(', ')}" unless disallowed_fields.empty?
missing_required_fields = (required_fields - result[:metadata].reject { |_, v| v.blank? }.keys)
unless missing_required_fields.empty?
result[:missing_required_fields] = "missing required fields: #{missing_required_fields.join(', ')}"
end
if required_fields.include?(:feature_categories)
object_feature_categories = result.dig(:metadata, :feature_categories)
unless feature_categories_valid?(object_feature_categories)
result[:invalid_feature_category] =
"invalid feature category: #{object_feature_categories}" \
"Please use a category from https://about.gitlab.com/handbook/product/categories/#categories-a-z"
end
end
rescue Psych::SyntaxError => ex
result[:error] = ex.message
end
result
end
# rubocop:disable Naming/HeredocDelimiterNaming
def object_metadata_errors(title, field, objects)
lines = objects.map do |object_name|
<<~EOM
#{object_metadata_file(object_name)}
#{metadata[object_name][field]}
EOM
end
multiline_error(title, lines)
end
def multiline_error(title, lines)
<<~EOM
#{title}:
#{lines.join("\n")}
EOM
end
# rubocop:enable Naming/HeredocDelimiterNaming
end
RSpec.describe 'Views documentation', feature_category: :database do
database_base_models = Gitlab::Database.database_base_models.select { |k, _| k != 'geo' }
views = database_base_models.flat_map { |_, m| m.connection.views }.sort.uniq
directory_path = File.join('db', 'docs', 'views')
required_fields = %i[feature_categories view_name gitlab_schema]
include_examples 'validate dictionary', views, directory_path, required_fields
end
RSpec.describe 'Tables documentation', feature_category: :database do
database_base_models = Gitlab::Database.database_base_models.select { |k, _| k != 'geo' }
tables = database_base_models.flat_map { |_, m| m.connection.tables }.sort.uniq
directory_path = File.join('db', 'docs')
required_fields = %i[feature_categories table_name gitlab_schema]
include_examples 'validate dictionary', tables, directory_path, required_fields
end
RSpec.describe 'Deleted tables documentation', feature_category: :database do
directory_path = File.join('db', 'docs', 'deleted_tables')
tables = Dir.glob(File.join(directory_path, '*.yml')).map { |f| File.basename(f, '.yml') }.sort.uniq
required_fields = %i[table_name gitlab_schema removed_by_url removed_in_milestone]
include_examples 'validate dictionary', tables, directory_path, required_fields
end
RSpec.describe 'Deleted views documentation', feature_category: :database do
directory_path = File.join('db', 'docs', 'deleted_views')
views = Dir.glob(File.join(directory_path, '*.yml')).map { |f| File.basename(f, '.yml') }.sort.uniq
required_fields = %i[view_name gitlab_schema removed_by_url removed_in_milestone]
include_examples 'validate dictionary', views, directory_path, required_fields
end
|
