diff options
Diffstat (limited to 'tooling')
| -rw-r--r-- | tooling/lib/tooling/helm3_client.rb | 111 | ||||
| -rw-r--r-- | tooling/lib/tooling/kubernetes_client.rb | 86 | ||||
| -rw-r--r-- | tooling/lib/tooling/test_file_finder.rb | 44 | ||||
| -rw-r--r-- | tooling/overcommit/Gemfile | 2 | ||||
| -rw-r--r-- | tooling/overcommit/Gemfile.lock | 4 |
5 files changed, 230 insertions, 17 deletions
diff --git a/tooling/lib/tooling/helm3_client.rb b/tooling/lib/tooling/helm3_client.rb new file mode 100644 index 00000000000..802ff9b9661 --- /dev/null +++ b/tooling/lib/tooling/helm3_client.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +require 'time' +require 'json' +require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen) +require_relative '../../../lib/gitlab/json' unless defined?(Gitlab::Json) + +module Tooling + class Helm3Client + CommandFailedError = Class.new(StandardError) + + attr_reader :namespace + + RELEASE_JSON_ATTRIBUTES = %w[name revision updated status chart app_version namespace].freeze + PAGINATION_SIZE = 256 # Default helm list pagination size + + Release = Struct.new(:name, :revision, :last_update, :status, :chart, :app_version, :namespace) do + def revision + @revision ||= self[:revision].to_i + end + + def last_update + @last_update ||= Time.parse(self[:last_update]) + end + end + + # A single page of data and the corresponding page number. + Page = Struct.new(:releases, :number) + + def initialize(namespace:) + @namespace = namespace + end + + def releases(args: []) + each_release(args) + end + + def delete(release_name:) + run_command([ + 'uninstall', + %(--namespace "#{namespace}"), + release_name + ]) + end + + private + + def run_command(command) + final_command = ['helm', *command].join(' ') + puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output + + result = Gitlab::Popen.popen_with_detail([final_command]) + + if result.status.success? + result.stdout.chomp.freeze + else + raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}" + end + end + + def raw_releases(page, args = []) + command = [ + 'list', + %(--namespace "#{namespace}"), + %(--max #{PAGINATION_SIZE}), + %(--offset #{PAGINATION_SIZE * page}), + %(--output json), + *args + ] + releases = Gitlab::Json.parse(run_command(command)) + + releases.map do |release| + Release.new(*release.values_at(*RELEASE_JSON_ATTRIBUTES)) + end + rescue ::JSON::ParserError => ex + puts "Ignoring this JSON parsing error: #{ex}" # rubocop:disable Rails/Output + [] + end + + # Fetches data from Helm and yields a Page object for every page + # of data, without loading all of them into memory. + # + # method - The Octokit method to use for getting the data. + # args - Arguments to pass to the `helm list` command. + def each_releases_page(args, &block) + return to_enum(__method__, args) unless block_given? + + page = 0 + final_args = args.dup + + begin + collection = raw_releases(page, final_args) + + yield Page.new(collection, page += 1) + end while collection.any? + end + + # Iterates over all of the releases. + # + # args - Any arguments to pass to the `helm list` command. + def each_release(args, &block) + return to_enum(__method__, args) unless block_given? + + each_releases_page(args) do |page| + page.releases.each do |release| + yield release + end + end + end + end +end diff --git a/tooling/lib/tooling/kubernetes_client.rb b/tooling/lib/tooling/kubernetes_client.rb new file mode 100644 index 00000000000..14b96addf87 --- /dev/null +++ b/tooling/lib/tooling/kubernetes_client.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen) +require_relative '../../../lib/gitlab/json' unless defined?(Gitlab::JSON) + +module Tooling + class KubernetesClient + RESOURCE_LIST = 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd' + CommandFailedError = Class.new(StandardError) + + attr_reader :namespace + + def initialize(namespace:) + @namespace = namespace + end + + def cleanup(release_name:, wait: true) + delete_by_selector(release_name: release_name, wait: wait) + delete_by_matching_name(release_name: release_name) + end + + private + + def delete_by_selector(release_name:, wait:) + selector = case release_name + when String + %(-l release="#{release_name}") + when Array + %(-l 'release in (#{release_name.join(', ')})') + else + raise ArgumentError, 'release_name must be a string or an array' + end + + command = [ + 'delete', + RESOURCE_LIST, + %(--namespace "#{namespace}"), + '--now', + '--ignore-not-found', + '--include-uninitialized', + %(--wait=#{wait}), + selector + ] + + run_command(command) + end + + def delete_by_matching_name(release_name:) + resource_names = raw_resource_names + command = [ + 'delete', + %(--namespace "#{namespace}"), + '--ignore-not-found' + ] + + Array(release_name).each do |release| + resource_names + .select { |resource_name| resource_name.include?(release) } + .each { |matching_resource| run_command(command + [matching_resource]) } + end + end + + def raw_resource_names + command = [ + 'get', + RESOURCE_LIST, + %(--namespace "#{namespace}"), + '-o name' + ] + run_command(command).lines.map(&:strip) + end + + def run_command(command) + final_command = ['kubectl', *command].join(' ') + puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output + + result = Gitlab::Popen.popen_with_detail([final_command]) + + if result.status.success? + result.stdout.chomp.freeze + else + raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}" + end + end + end +end diff --git a/tooling/lib/tooling/test_file_finder.rb b/tooling/lib/tooling/test_file_finder.rb index 4cf7ad35922..cf5de190c4a 100644 --- a/tooling/lib/tooling/test_file_finder.rb +++ b/tooling/lib/tooling/test_file_finder.rb @@ -12,7 +12,7 @@ module Tooling end def test_files - impacted_tests = ee_impact | non_ee_impact + impacted_tests = ee_impact | non_ee_impact | either_impact impacted_tests.impact(@file) end @@ -23,20 +23,22 @@ module Tooling class ImpactedTestFile attr_reader :pattern_matchers - def initialize + def initialize(prefix: nil) @pattern_matchers = {} + @prefix = prefix yield self if block_given? end def associate(pattern, &block) - @pattern_matchers[pattern] = block + @pattern_matchers[%r{^#{@prefix}#{pattern}}] = block end def impact(file) @pattern_matchers.each_with_object(Set.new) do |(pattern, block), result| if (match = pattern.match(file)) - result << block.call(match) + test_files = block.call(match) + result.merge(Array(test_files)) end end.to_a end @@ -54,24 +56,38 @@ module Tooling end def ee_impact - ImpactedTestFile.new do |impact| + ImpactedTestFile.new(prefix: EE_PREFIX) do |impact| unless foss_test_only - impact.associate(%r{^#{EE_PREFIX}app/(.+)\.rb$}) { |match| "#{EE_PREFIX}spec/#{match[1]}_spec.rb" } - impact.associate(%r{^#{EE_PREFIX}app/(.*/)ee/(.+)\.rb$}) { |match| "#{EE_PREFIX}spec/#{match[1]}#{match[2]}_spec.rb" } - impact.associate(%r{^#{EE_PREFIX}lib/(.+)\.rb$}) { |match| "#{EE_PREFIX}spec/lib/#{match[1]}_spec.rb" } - impact.associate(%r{^#{EE_PREFIX}spec/(.+)_spec.rb$}) { |match| match[0] } + impact.associate(%r{app/(.+)\.rb$}) { |match| "#{EE_PREFIX}spec/#{match[1]}_spec.rb" } + impact.associate(%r{app/(.*/)ee/(.+)\.rb$}) { |match| "#{EE_PREFIX}spec/#{match[1]}#{match[2]}_spec.rb" } + impact.associate(%r{lib/(.+)\.rb$}) { |match| "#{EE_PREFIX}spec/lib/#{match[1]}_spec.rb" } end - impact.associate(%r{^#{EE_PREFIX}(?!spec)(.*/)ee/(.+)\.rb$}) { |match| "spec/#{match[1]}#{match[2]}_spec.rb" } - impact.associate(%r{^#{EE_PREFIX}spec/(.*/)ee/(.+)\.rb$}) { |match| "spec/#{match[1]}#{match[2]}.rb" } + impact.associate(%r{(?!spec)(.*/)ee/(.+)\.rb$}) { |match| "spec/#{match[1]}#{match[2]}_spec.rb" } + impact.associate(%r{spec/(.*/)ee/(.+)\.rb$}) { |match| "spec/#{match[1]}#{match[2]}.rb" } end end def non_ee_impact ImpactedTestFile.new do |impact| - impact.associate(%r{^app/(.+)\.rb$}) { |match| "spec/#{match[1]}_spec.rb" } - impact.associate(%r{^(tooling/)?lib/(.+)\.rb$}) { |match| "spec/#{match[1]}lib/#{match[2]}_spec.rb" } - impact.associate(%r{^spec/(.+)_spec.rb$}) { |match| match[0] } + impact.associate(%r{app/(.+)\.rb$}) { |match| "spec/#{match[1]}_spec.rb" } + impact.associate(%r{(tooling/)?lib/(.+)\.rb$}) { |match| "spec/#{match[1]}lib/#{match[2]}_spec.rb" } + impact.associate(%r{config/initializers/(.+)\.rb$}) { |match| "spec/initializers/#{match[1]}_spec.rb" } + impact.associate('db/structure.sql') { 'spec/db/schema_spec.rb' } + impact.associate(%r{db/(?:post_)?migrate/([0-9]+)_(.+)\.rb$}) do |match| + [ + "spec/migrations/#{match[2]}_spec.rb", + "spec/migrations/#{match[1]}_#{match[2]}_spec.rb" + ] + end + end + end + + def either_impact + ImpactedTestFile.new(prefix: %r{^(?<prefix>#{EE_PREFIX})?}) do |impact| + impact.associate(%r{app/views/(?<view>.+)\.haml$}) { |match| "#{match[:prefix]}spec/views/#{match[:view]}.haml_spec.rb" } + impact.associate(%r{spec/(.+)_spec\.rb$}) { |match| match[0] } + impact.associate(%r{spec/factories/.+\.rb$}) { 'spec/factories_spec.rb' } end end end diff --git a/tooling/overcommit/Gemfile b/tooling/overcommit/Gemfile index 120cb1ad8d0..615da316fd5 100644 --- a/tooling/overcommit/Gemfile +++ b/tooling/overcommit/Gemfile @@ -4,6 +4,6 @@ source 'https://rubygems.org' gem 'overcommit' -gem 'gitlab-styles', '~> 4.2.0', require: false +gem 'gitlab-styles', '~> 4.3.0', require: false gem 'scss_lint', '~> 0.56.0', require: false gem 'haml_lint', '~> 0.34.0', require: false diff --git a/tooling/overcommit/Gemfile.lock b/tooling/overcommit/Gemfile.lock index 7ab10f741ed..d3a98855d96 100644 --- a/tooling/overcommit/Gemfile.lock +++ b/tooling/overcommit/Gemfile.lock @@ -11,7 +11,7 @@ GEM childprocess (3.0.0) concurrent-ruby (1.1.6) ffi (1.12.2) - gitlab-styles (4.2.0) + gitlab-styles (4.3.0) rubocop (~> 0.82.0) rubocop-gitlab-security (~> 0.1.0) rubocop-performance (~> 1.5.2) @@ -83,7 +83,7 @@ PLATFORMS ruby DEPENDENCIES - gitlab-styles (~> 4.2.0) + gitlab-styles (~> 4.3.0) haml_lint (~> 0.34.0) overcommit scss_lint (~> 0.56.0) |