summaryrefslogtreecommitdiff
path: root/tooling
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-07-20 12:26:25 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2020-07-20 12:26:25 +0000
commita09983ae35713f5a2bbb100981116d31ce99826e (patch)
tree2ee2af7bd104d57086db360a7e6d8c9d5d43667a /tooling
parent18c5ab32b738c0b6ecb4d0df3994000482f34bd8 (diff)
downloadgitlab-ce-a09983ae35713f5a2bbb100981116d31ce99826e.tar.gz
Add latest changes from gitlab-org/gitlab@13-2-stable-ee
Diffstat (limited to 'tooling')
-rw-r--r--tooling/lib/tooling/helm3_client.rb111
-rw-r--r--tooling/lib/tooling/kubernetes_client.rb86
-rw-r--r--tooling/lib/tooling/test_file_finder.rb44
-rw-r--r--tooling/overcommit/Gemfile2
-rw-r--r--tooling/overcommit/Gemfile.lock4
5 files changed, 230 insertions, 17 deletions
diff --git a/tooling/lib/tooling/helm3_client.rb b/tooling/lib/tooling/helm3_client.rb
new file mode 100644
index 00000000000..802ff9b9661
--- /dev/null
+++ b/tooling/lib/tooling/helm3_client.rb
@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'time'
+require 'json'
+require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen)
+require_relative '../../../lib/gitlab/json' unless defined?(Gitlab::Json)
+
+module Tooling
+ class Helm3Client
+ CommandFailedError = Class.new(StandardError)
+
+ attr_reader :namespace
+
+ RELEASE_JSON_ATTRIBUTES = %w[name revision updated status chart app_version namespace].freeze
+ PAGINATION_SIZE = 256 # Default helm list pagination size
+
+ Release = Struct.new(:name, :revision, :last_update, :status, :chart, :app_version, :namespace) do
+ def revision
+ @revision ||= self[:revision].to_i
+ end
+
+ def last_update
+ @last_update ||= Time.parse(self[:last_update])
+ end
+ end
+
+ # A single page of data and the corresponding page number.
+ Page = Struct.new(:releases, :number)
+
+ def initialize(namespace:)
+ @namespace = namespace
+ end
+
+ def releases(args: [])
+ each_release(args)
+ end
+
+ def delete(release_name:)
+ run_command([
+ 'uninstall',
+ %(--namespace "#{namespace}"),
+ release_name
+ ])
+ end
+
+ private
+
+ def run_command(command)
+ final_command = ['helm', *command].join(' ')
+ puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
+
+ result = Gitlab::Popen.popen_with_detail([final_command])
+
+ if result.status.success?
+ result.stdout.chomp.freeze
+ else
+ raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
+ end
+ end
+
+ def raw_releases(page, args = [])
+ command = [
+ 'list',
+ %(--namespace "#{namespace}"),
+ %(--max #{PAGINATION_SIZE}),
+ %(--offset #{PAGINATION_SIZE * page}),
+ %(--output json),
+ *args
+ ]
+ releases = Gitlab::Json.parse(run_command(command))
+
+ releases.map do |release|
+ Release.new(*release.values_at(*RELEASE_JSON_ATTRIBUTES))
+ end
+ rescue ::JSON::ParserError => ex
+ puts "Ignoring this JSON parsing error: #{ex}" # rubocop:disable Rails/Output
+ []
+ end
+
+ # Fetches data from Helm and yields a Page object for every page
+ # of data, without loading all of them into memory.
+ #
+ # method - The Octokit method to use for getting the data.
+ # args - Arguments to pass to the `helm list` command.
+ def each_releases_page(args, &block)
+ return to_enum(__method__, args) unless block_given?
+
+ page = 0
+ final_args = args.dup
+
+ begin
+ collection = raw_releases(page, final_args)
+
+ yield Page.new(collection, page += 1)
+ end while collection.any?
+ end
+
+ # Iterates over all of the releases.
+ #
+ # args - Any arguments to pass to the `helm list` command.
+ def each_release(args, &block)
+ return to_enum(__method__, args) unless block_given?
+
+ each_releases_page(args) do |page|
+ page.releases.each do |release|
+ yield release
+ end
+ end
+ end
+ end
+end
diff --git a/tooling/lib/tooling/kubernetes_client.rb b/tooling/lib/tooling/kubernetes_client.rb
new file mode 100644
index 00000000000..14b96addf87
--- /dev/null
+++ b/tooling/lib/tooling/kubernetes_client.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen)
+require_relative '../../../lib/gitlab/json' unless defined?(Gitlab::JSON)
+
+module Tooling
+ class KubernetesClient
+ RESOURCE_LIST = 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd'
+ CommandFailedError = Class.new(StandardError)
+
+ attr_reader :namespace
+
+ def initialize(namespace:)
+ @namespace = namespace
+ end
+
+ def cleanup(release_name:, wait: true)
+ delete_by_selector(release_name: release_name, wait: wait)
+ delete_by_matching_name(release_name: release_name)
+ end
+
+ private
+
+ def delete_by_selector(release_name:, wait:)
+ selector = case release_name
+ when String
+ %(-l release="#{release_name}")
+ when Array
+ %(-l 'release in (#{release_name.join(', ')})')
+ else
+ raise ArgumentError, 'release_name must be a string or an array'
+ end
+
+ command = [
+ 'delete',
+ RESOURCE_LIST,
+ %(--namespace "#{namespace}"),
+ '--now',
+ '--ignore-not-found',
+ '--include-uninitialized',
+ %(--wait=#{wait}),
+ selector
+ ]
+
+ run_command(command)
+ end
+
+ def delete_by_matching_name(release_name:)
+ resource_names = raw_resource_names
+ command = [
+ 'delete',
+ %(--namespace "#{namespace}"),
+ '--ignore-not-found'
+ ]
+
+ Array(release_name).each do |release|
+ resource_names
+ .select { |resource_name| resource_name.include?(release) }
+ .each { |matching_resource| run_command(command + [matching_resource]) }
+ end
+ end
+
+ def raw_resource_names
+ command = [
+ 'get',
+ RESOURCE_LIST,
+ %(--namespace "#{namespace}"),
+ '-o name'
+ ]
+ run_command(command).lines.map(&:strip)
+ end
+
+ def run_command(command)
+ final_command = ['kubectl', *command].join(' ')
+ puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
+
+ result = Gitlab::Popen.popen_with_detail([final_command])
+
+ if result.status.success?
+ result.stdout.chomp.freeze
+ else
+ raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
+ end
+ end
+ end
+end
diff --git a/tooling/lib/tooling/test_file_finder.rb b/tooling/lib/tooling/test_file_finder.rb
index 4cf7ad35922..cf5de190c4a 100644
--- a/tooling/lib/tooling/test_file_finder.rb
+++ b/tooling/lib/tooling/test_file_finder.rb
@@ -12,7 +12,7 @@ module Tooling
end
def test_files
- impacted_tests = ee_impact | non_ee_impact
+ impacted_tests = ee_impact | non_ee_impact | either_impact
impacted_tests.impact(@file)
end
@@ -23,20 +23,22 @@ module Tooling
class ImpactedTestFile
attr_reader :pattern_matchers
- def initialize
+ def initialize(prefix: nil)
@pattern_matchers = {}
+ @prefix = prefix
yield self if block_given?
end
def associate(pattern, &block)
- @pattern_matchers[pattern] = block
+ @pattern_matchers[%r{^#{@prefix}#{pattern}}] = block
end
def impact(file)
@pattern_matchers.each_with_object(Set.new) do |(pattern, block), result|
if (match = pattern.match(file))
- result << block.call(match)
+ test_files = block.call(match)
+ result.merge(Array(test_files))
end
end.to_a
end
@@ -54,24 +56,38 @@ module Tooling
end
def ee_impact
- ImpactedTestFile.new do |impact|
+ ImpactedTestFile.new(prefix: EE_PREFIX) do |impact|
unless foss_test_only
- impact.associate(%r{^#{EE_PREFIX}app/(.+)\.rb$}) { |match| "#{EE_PREFIX}spec/#{match[1]}_spec.rb" }
- impact.associate(%r{^#{EE_PREFIX}app/(.*/)ee/(.+)\.rb$}) { |match| "#{EE_PREFIX}spec/#{match[1]}#{match[2]}_spec.rb" }
- impact.associate(%r{^#{EE_PREFIX}lib/(.+)\.rb$}) { |match| "#{EE_PREFIX}spec/lib/#{match[1]}_spec.rb" }
- impact.associate(%r{^#{EE_PREFIX}spec/(.+)_spec.rb$}) { |match| match[0] }
+ impact.associate(%r{app/(.+)\.rb$}) { |match| "#{EE_PREFIX}spec/#{match[1]}_spec.rb" }
+ impact.associate(%r{app/(.*/)ee/(.+)\.rb$}) { |match| "#{EE_PREFIX}spec/#{match[1]}#{match[2]}_spec.rb" }
+ impact.associate(%r{lib/(.+)\.rb$}) { |match| "#{EE_PREFIX}spec/lib/#{match[1]}_spec.rb" }
end
- impact.associate(%r{^#{EE_PREFIX}(?!spec)(.*/)ee/(.+)\.rb$}) { |match| "spec/#{match[1]}#{match[2]}_spec.rb" }
- impact.associate(%r{^#{EE_PREFIX}spec/(.*/)ee/(.+)\.rb$}) { |match| "spec/#{match[1]}#{match[2]}.rb" }
+ impact.associate(%r{(?!spec)(.*/)ee/(.+)\.rb$}) { |match| "spec/#{match[1]}#{match[2]}_spec.rb" }
+ impact.associate(%r{spec/(.*/)ee/(.+)\.rb$}) { |match| "spec/#{match[1]}#{match[2]}.rb" }
end
end
def non_ee_impact
ImpactedTestFile.new do |impact|
- impact.associate(%r{^app/(.+)\.rb$}) { |match| "spec/#{match[1]}_spec.rb" }
- impact.associate(%r{^(tooling/)?lib/(.+)\.rb$}) { |match| "spec/#{match[1]}lib/#{match[2]}_spec.rb" }
- impact.associate(%r{^spec/(.+)_spec.rb$}) { |match| match[0] }
+ impact.associate(%r{app/(.+)\.rb$}) { |match| "spec/#{match[1]}_spec.rb" }
+ impact.associate(%r{(tooling/)?lib/(.+)\.rb$}) { |match| "spec/#{match[1]}lib/#{match[2]}_spec.rb" }
+ impact.associate(%r{config/initializers/(.+)\.rb$}) { |match| "spec/initializers/#{match[1]}_spec.rb" }
+ impact.associate('db/structure.sql') { 'spec/db/schema_spec.rb' }
+ impact.associate(%r{db/(?:post_)?migrate/([0-9]+)_(.+)\.rb$}) do |match|
+ [
+ "spec/migrations/#{match[2]}_spec.rb",
+ "spec/migrations/#{match[1]}_#{match[2]}_spec.rb"
+ ]
+ end
+ end
+ end
+
+ def either_impact
+ ImpactedTestFile.new(prefix: %r{^(?<prefix>#{EE_PREFIX})?}) do |impact|
+ impact.associate(%r{app/views/(?<view>.+)\.haml$}) { |match| "#{match[:prefix]}spec/views/#{match[:view]}.haml_spec.rb" }
+ impact.associate(%r{spec/(.+)_spec\.rb$}) { |match| match[0] }
+ impact.associate(%r{spec/factories/.+\.rb$}) { 'spec/factories_spec.rb' }
end
end
end
diff --git a/tooling/overcommit/Gemfile b/tooling/overcommit/Gemfile
index 120cb1ad8d0..615da316fd5 100644
--- a/tooling/overcommit/Gemfile
+++ b/tooling/overcommit/Gemfile
@@ -4,6 +4,6 @@
source 'https://rubygems.org'
gem 'overcommit'
-gem 'gitlab-styles', '~> 4.2.0', require: false
+gem 'gitlab-styles', '~> 4.3.0', require: false
gem 'scss_lint', '~> 0.56.0', require: false
gem 'haml_lint', '~> 0.34.0', require: false
diff --git a/tooling/overcommit/Gemfile.lock b/tooling/overcommit/Gemfile.lock
index 7ab10f741ed..d3a98855d96 100644
--- a/tooling/overcommit/Gemfile.lock
+++ b/tooling/overcommit/Gemfile.lock
@@ -11,7 +11,7 @@ GEM
childprocess (3.0.0)
concurrent-ruby (1.1.6)
ffi (1.12.2)
- gitlab-styles (4.2.0)
+ gitlab-styles (4.3.0)
rubocop (~> 0.82.0)
rubocop-gitlab-security (~> 0.1.0)
rubocop-performance (~> 1.5.2)
@@ -83,7 +83,7 @@ PLATFORMS
ruby
DEPENDENCIES
- gitlab-styles (~> 4.2.0)
+ gitlab-styles (~> 4.3.0)
haml_lint (~> 0.34.0)
overcommit
scss_lint (~> 0.56.0)