diff options
Diffstat (limited to 'qa/qa/tools/test_resources_handler.rb')
| -rw-r--r-- | qa/qa/tools/test_resources_handler.rb | 105 |
1 files changed, 71 insertions, 34 deletions
diff --git a/qa/qa/tools/test_resources_handler.rb b/qa/qa/tools/test_resources_handler.rb index 60c6dbfc16c..068fe37a37b 100644 --- a/qa/qa/tools/test_resources_handler.rb +++ b/qa/qa/tools/test_resources_handler.rb @@ -25,13 +25,17 @@ module QA module Tools class TestResourcesHandler include Support::API - - IGNORED_RESOURCES = [ - 'QA::Resource::CiVariable', - 'QA::Resource::Repository::Commit', - 'QA::EE::Resource::GroupIteration', - 'QA::EE::Resource::Settings::Elasticsearch', - 'QA::EE::Resource::VulnerabilityItem' + include Ci::Helpers + + IGNORED_RESOURCES = %w[ + QA::Resource::CiVariable + QA::Resource::Repository::Commit + QA::Resource::Design + QA::EE::Resource::GroupIteration + QA::EE::Resource::Settings::Elasticsearch + QA::EE::Resource::VulnerabilityItem + QA::EE::Resource::ScanResultPolicyProject + QA::EE::Resource::ScanResultPolicyCommit ].freeze PROJECT = 'gitlab-qa-resources' @@ -44,10 +48,19 @@ module QA def run_delete failures = files.flat_map do |file| resources = read_file(file) - next if resources.nil? + if resources.nil? + logger.info("#{file} is empty, next...") + next + end filtered_resources = filter_resources(resources) + if filtered_resources.nil? + logger.info("No resources left to delete after filtering!") + next + end + delete_resources(filtered_resources) + delete_groups_permanently(filtered_resources['QA::Resource::Group']) end return puts "\nDone" if failures.empty? @@ -62,17 +75,15 @@ module QA # E.g: staging/failed-test-resources-<randomhex>.json def upload(ci_project_name) if files.empty? - puts "\nNothing to upload!" - exit 0 + logger.info("\nNothing to upload!") + return end files.each do |file| file_name = "#{ci_project_name}/#{file.split('/').last}" - Runtime::Logger.info("Uploading #{file_name}...") + logger.info("Uploading #{file_name}...") gcs_storage.put_object(BUCKET, file_name, File.read(file)) end - - puts "\nDone" end # Download files from GCS bucket by environment name @@ -85,40 +96,38 @@ module QA end if files_list.blank? - puts "\nNothing to download!" - exit 0 + logger.info("\nNothing to download!") + return end FileUtils.mkdir_p('tmp/') files_list.each do |file_name| local_path = "tmp/#{file_name.split('/').last}" - Runtime::Logger.info("Downloading #{file_name} to #{local_path}") + logger.info("Downloading #{file_name} to #{local_path}") file = gcs_storage.get_object(BUCKET, file_name) File.write(local_path, file[:body]) - Runtime::Logger.info("Deleting #{file_name} from bucket") + logger.info("Deleting #{file_name} from bucket") gcs_storage.delete_object(BUCKET, file_name) end - - puts "\nDone" end private def files - Runtime::Logger.info('Gathering JSON files...') + logger.info('Gathering JSON files...') files = Dir.glob(@file_pattern) if files.empty? - puts "There is no file with this pattern #{@file_pattern}" + logger.info("There is no file with this pattern #{@file_pattern}") exit 0 end files.reject! { |file| File.zero?(file) } if files.empty? - puts "\nAll files were empty and rejected, nothing more to do!" + logger.info("\nAll files were empty and rejected, nothing more to do!") exit 0 end @@ -126,14 +135,15 @@ module QA end def read_file(file) + logger.info("Reading and processing #{file}...") JSON.parse(File.read(file)) rescue JSON::ParserError - Runtime::Logger.error("Failed to read #{file} - Invalid format") + logger.error("Failed to read #{file} - Invalid format") nil end def filter_resources(resources) - Runtime::Logger.info('Filtering resources - Only keep deletable resources...') + logger.info('Filtering resources - Only keep deletable resources...') transformed_values = resources.transform_values! do |v| v.reject do |attributes| @@ -147,28 +157,55 @@ module QA end def delete_resources(resources) - if resources.nil? - puts "\nNo resources left to delete after filtering!" - exit 0 - end - resources.each_with_object([]) do |(key, value), failures| value.each do |resource| - next if resource_not_found?(resource['api_path']) - resource_info = resource['info'] ? "#{key} - #{resource['info']}" : "#{key} at #{resource['api_path']}" + logger.info("Processing #{resource_info}...") + + if resource_not_found?(resource['api_path']) + logger.info("#{resource['api_path']} returns 404, next...") + next + end + delete_response = delete(Runtime::API::Request.new(api_client, resource['api_path']).url) if delete_response.code == 202 || delete_response.code == 204 - Runtime::Logger.info("Deleting #{resource_info}... SUCCESS") + if key == 'QA::Resource::Group' && !resource_not_found?(resource['api_path']) + logger.info("Successfully marked #{resource_info} for deletion...") + else + logger.info("Deleting #{resource_info}... \e[32mSUCCESS\e[0m") + end else - Runtime::Logger.info("Deleting #{resource_info}... FAILED - #{delete_response}") - failures << resource_info + logger.info("Deleting #{resource_info}... \e[31mFAILED - #{delete_response}\e[0m") + # We might try to delete some groups already marked for deletion, it's fine to ignore these failures + failures << resource_info unless key == 'QA::Resource::Group' end end end end + def delete_groups_permanently(groups) + groups.each_with_object([]) do |group, failures| + logger.info("Processing QA::Resource::Group #{group['info']}...") + + if resource_not_found?(group['api_path']) + logger.info("#{group['api_path']} returns 404, next...") + next + end + + permanent_delete_path = "#{group['api_path']}?permanently_remove=true"\ + "&full_path=#{group['info'].split("'").last}" + response = delete(Runtime::API::Request.new(api_client, permanent_delete_path).url) + + if response.code == 202 + logger.info("Permanently deleting group #{group['info']}... \e[32mSUCCESS\e[0m") + else + logger.info("Permanently deleting group #{group['info']}... \e[31mFAILED - #{response}\e[0m") + failures << "QA::Resource::Group #{group['info']}" + end + end + end + def resource_not_found?(api_path) # if api path contains param "?hard_delete=<boolean>", remove it get(Runtime::API::Request.new(api_client, api_path.split('?').first).url).code.eql? 404 |