1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
|
# frozen_string_literal: true
require "fog/google"
# This script handles resources created during E2E test runs
#
# Delete: find all matching file pattern, read file and delete resources
# rake test_resources:delete[<file_pattern>]
#
# Upload: find all matching file pattern for failed test resources
# upload these files to GCS bucket `failed-test-resources` under specific environment name
# rake test_resources:upload[<file_pattern>,<ci_project_name>]
#
# Download: download JSON files under a given environment name (bucket directory)
# save to local under `tmp/`
# rake test_resources:download[<ci_project_name>]
#
# Required environment variables:
# GITLAB_ADDRESS, required for delete task
# GITLAB_QA_ACCESS_TOKEN, required for delete task
# QA_TEST_RESOURCES_FILE_PATTERN, optional for delete task, required for upload task
# QA_FAILED_TEST_RESOURCES_GCS_CREDENTIALS, required for upload task or download task
module QA
module Tools
class TestResourcesHandler
include Support::API
IGNORED_RESOURCES = [
'QA::Resource::PersonalAccessToken',
'QA::Resource::CiVariable',
'QA::Resource::Repository::Commit',
'QA::EE::Resource::GroupIteration',
'QA::EE::Resource::Settings::Elasticsearch'
].freeze
PROJECT = 'gitlab-qa-resources'
BUCKET = 'failed-test-resources'
def initialize(file_pattern = nil)
@file_pattern = file_pattern
end
def run_delete
failures = files.flat_map do |file|
resources = read_file(file)
next if resources.nil?
filtered_resources = filter_resources(resources)
delete_resources(filtered_resources)
end
return puts "\nDone" if failures.empty?
puts "\nFailed to delete #{failures.size} resources:\n"
puts failures
end
# Upload resources from failed test suites to GCS bucket
# Files are organized by environment in which tests were executed
#
# E.g: staging/failed-test-resources-<randomhex>.json
def upload(ci_project_name)
return puts "\nNothing to upload!" if files.empty?
files.each do |file|
file_name = "#{ci_project_name}/#{file.split('/').last}"
Runtime::Logger.info("Uploading #{file_name}...")
gcs_storage.put_object(BUCKET, file_name, File.read(file))
end
puts "\nDone"
end
# Download files from GCS bucket by environment name
# Delete the files afterward
def download(ci_project_name)
bucket_items = gcs_storage.list_objects(BUCKET, prefix: ci_project_name).items
files_list = bucket_items&.each_with_object([]) do |obj, arr|
arr << obj.name
end
return puts "\nNothing to download!" if files_list.blank?
FileUtils.mkdir_p('tmp/')
files_list.each do |file_name|
local_path = "tmp/#{file_name.split('/').last}"
Runtime::Logger.info("Downloading #{file_name} to #{local_path}")
file = gcs_storage.get_object(BUCKET, file_name)
File.write(local_path, file[:body])
Runtime::Logger.info("Deleting #{file_name} from bucket")
gcs_storage.delete_object(BUCKET, file_name)
end
puts "\nDone"
end
private
def files
Runtime::Logger.info('Gathering JSON files...')
files = Dir.glob(@file_pattern)
abort("There is no file with this pattern #{@file_pattern}") if files.empty?
files.reject! { |file| File.zero?(file) }
files
end
def read_file(file)
JSON.parse(File.read(file))
rescue JSON::ParserError
Runtime::Logger.error("Failed to read #{file} - Invalid format")
nil
end
def filter_resources(resources)
Runtime::Logger.info('Filtering resources - Only keep deletable resources...')
transformed_values = resources.transform_values! do |v|
v.reject do |attributes|
attributes['info'] == "with full_path 'gitlab-qa-sandbox-group'" ||
attributes['http_method'] == 'get' && !attributes['info']&.include?("with username 'qa-") ||
attributes['api_path'] == 'Cannot find resource API path'
end
end
transformed_values.reject! { |k, v| v.empty? || IGNORED_RESOURCES.include?(k) }
end
def delete_resources(resources)
Runtime::Logger.info('Nothing to delete.') && return if resources.nil?
resources.each_with_object([]) do |(key, value), failures|
value.each do |resource|
next if resource_not_found?(resource['api_path'])
resource_info = resource['info'] ? "#{key} - #{resource['info']}" : "#{key} at #{resource['api_path']}"
delete_response = delete(Runtime::API::Request.new(api_client, resource['api_path']).url)
if delete_response.code == 202 || delete_response.code == 204
Runtime::Logger.info("Deleting #{resource_info}... SUCCESS")
else
Runtime::Logger.info("Deleting #{resource_info}... FAILED")
failures << resource_info
end
end
end
end
def resource_not_found?(api_path)
# if api path contains param "?hard_delete=<boolean>", remove it
get(Runtime::API::Request.new(api_client, api_path.split('?').first).url).code.eql? 404
end
def api_client
abort("\nPlease provide GITLAB_ADDRESS") unless ENV['GITLAB_ADDRESS']
abort("\nPlease provide GITLAB_QA_ACCESS_TOKEN") unless ENV['GITLAB_QA_ACCESS_TOKEN']
@api_client ||= Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['GITLAB_QA_ACCESS_TOKEN'])
end
def gcs_storage
@gcs_storage ||= Fog::Storage::Google.new(
google_project: PROJECT,
**(File.exist?(json_key) ? { google_json_key_location: json_key } : { google_json_key_string: json_key })
)
rescue StandardError => e
abort("\nThere might be something wrong with the JSON key file - [ERROR] #{e}")
end
# Path to GCS service account json key file
# Or the content of the key file as a hash
def json_key
abort("\nPlease provide QA_FAILED_TEST_RESOURCES_GCS_CREDENTIALS") unless ENV['QA_FAILED_TEST_RESOURCES_GCS_CREDENTIALS']
@json_key ||= ENV["QA_FAILED_TEST_RESOURCES_GCS_CREDENTIALS"]
end
end
end
end
|
