From 4555e1b21c365ed8303ffb7a3325d773c9b8bf31 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 19 May 2021 15:44:42 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-12-stable-ee --- app/models/clusters/agent.rb | 1 + app/models/clusters/agent_token.rb | 4 +- app/models/clusters/applications/elastic_stack.rb | 48 ++++++++-------------- app/models/clusters/applications/prometheus.rb | 33 +++++---------- app/models/clusters/applications/runner.rb | 2 +- app/models/clusters/cluster.rb | 24 ++++++++++- app/models/clusters/concerns/application_core.rb | 22 ++-------- .../clusters/concerns/elasticsearch_client.rb | 38 +++++++++++++++++ app/models/clusters/concerns/kubernetes_logger.rb | 27 ++++++++++++ app/models/clusters/integrations/elastic_stack.rb | 38 +++++++++++++++++ app/models/clusters/integrations/prometheus.rb | 38 +++++++++++++++++ app/models/clusters/providers/aws.rb | 2 +- 12 files changed, 199 insertions(+), 78 deletions(-) create mode 100644 app/models/clusters/concerns/elasticsearch_client.rb create mode 100644 app/models/clusters/concerns/kubernetes_logger.rb create mode 100644 app/models/clusters/integrations/elastic_stack.rb (limited to 'app/models/clusters') diff --git a/app/models/clusters/agent.rb b/app/models/clusters/agent.rb index c5b9dddb1da..9fb8cd024c5 100644 --- a/app/models/clusters/agent.rb +++ b/app/models/clusters/agent.rb @@ -8,6 +8,7 @@ module Clusters belongs_to :project, class_name: '::Project' # Otherwise, it will load ::Clusters::Project has_many :agent_tokens, class_name: 'Clusters::AgentToken' + has_many :last_used_agent_tokens, -> { order_last_used_at_desc }, class_name: 'Clusters::AgentToken', inverse_of: :agent scope :ordered_by_name, -> { order(:name) } scope :with_name, -> (name) { where(name: name) } diff --git a/app/models/clusters/agent_token.rb b/app/models/clusters/agent_token.rb index d42279502c5..27a3cd8d13d 100644 --- a/app/models/clusters/agent_token.rb +++ b/app/models/clusters/agent_token.rb @@ -6,7 +6,7 @@ module Clusters include TokenAuthenticatable add_authentication_token_field :token, encrypted: :required, token_generator: -> { Devise.friendly_token(50) } - cached_attr_reader :last_contacted_at + cached_attr_reader :last_used_at self.table_name = 'cluster_agent_tokens' @@ -21,6 +21,8 @@ module Clusters validates :description, length: { maximum: 1024 } validates :name, presence: true, length: { maximum: 255 } + scope :order_last_used_at_desc, -> { order(::Gitlab::Database.nulls_last_order('last_used_at', 'DESC')) } + def track_usage track_values = { last_used_at: Time.current.utc } diff --git a/app/models/clusters/applications/elastic_stack.rb b/app/models/clusters/applications/elastic_stack.rb index db18a29ec84..73c731aab1a 100644 --- a/app/models/clusters/applications/elastic_stack.rb +++ b/app/models/clusters/applications/elastic_stack.rb @@ -3,9 +3,9 @@ module Clusters module Applications class ElasticStack < ApplicationRecord - VERSION = '3.0.0' + include ::Clusters::Concerns::ElasticsearchClient - ELASTICSEARCH_PORT = 9200 + VERSION = '3.0.0' self.table_name = 'clusters_applications_elastic_stacks' @@ -13,10 +13,23 @@ module Clusters include ::Clusters::Concerns::ApplicationStatus include ::Clusters::Concerns::ApplicationVersion include ::Clusters::Concerns::ApplicationData - include ::Gitlab::Utils::StrongMemoize default_value_for :version, VERSION + after_destroy do + cluster&.find_or_build_integration_elastic_stack&.update(enabled: false, chart_version: nil) + end + + state_machine :status do + after_transition any => [:installed] do |application| + application.cluster&.find_or_build_integration_elastic_stack&.update(enabled: true, chart_version: application.version) + end + + after_transition any => [:uninstalled] do |application| + application.cluster&.find_or_build_integration_elastic_stack&.update(enabled: false, chart_version: nil) + end + end + def chart 'elastic-stack/elastic-stack' end @@ -51,31 +64,6 @@ module Clusters super.merge('wait-for-elasticsearch.sh': File.read("#{Rails.root}/vendor/elastic_stack/wait-for-elasticsearch.sh")) end - def elasticsearch_client(timeout: nil) - strong_memoize(:elasticsearch_client) do - next unless kube_client - - proxy_url = kube_client.proxy_url('service', service_name, ::Clusters::Applications::ElasticStack::ELASTICSEARCH_PORT, Gitlab::Kubernetes::Helm::NAMESPACE) - - Elasticsearch::Client.new(url: proxy_url) do |faraday| - # ensures headers containing auth data are appended to original client options - faraday.headers.merge!(kube_client.headers) - # ensure TLS certs are properly verified - faraday.ssl[:verify] = kube_client.ssl_options[:verify_ssl] - faraday.ssl[:cert_store] = kube_client.ssl_options[:cert_store] - faraday.options.timeout = timeout unless timeout.nil? - end - - rescue Kubeclient::HttpError => error - # If users have mistakenly set parameters or removed the depended clusters, - # `proxy_url` could raise an exception because gitlab can not communicate with the cluster. - # We check for a nil client in downstream use and behaviour is equivalent to an empty state - log_exception(error, :failed_to_create_elasticsearch_client) - - nil - end - end - def chart_above_v2? Gem::Version.new(version) >= Gem::Version.new('2.0.0') end @@ -106,10 +94,6 @@ module Clusters ] end - def kube_client - cluster&.kubeclient&.core_client - end - def migrate_to_3_script return [] if !updating? || chart_above_v3? diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index b9c136abab4..21f7e410843 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -22,21 +22,18 @@ module Clusters attr_encrypted :alert_manager_token, mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base_truncated, + key: Settings.attr_encrypted_db_key_base_32, algorithm: 'aes-256-gcm' + default_value_for(:alert_manager_token) { SecureRandom.hex } + after_destroy do - run_after_commit do - disable_prometheus_integration - end + cluster.find_or_build_integration_prometheus.destroy end state_machine :status do after_transition any => [:installed, :externally_installed] do |application| - application.run_after_commit do - Clusters::Applications::ActivateServiceWorker - .perform_async(application.cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass - end + application.cluster.find_or_build_integration_prometheus.update(enabled: true, alert_manager_token: application.alert_manager_token) end after_transition any => :updating do |application| @@ -44,6 +41,10 @@ module Clusters end end + def managed_prometheus? + !externally_installed? && !uninstalled? + end + def updated_since?(timestamp) last_update_started_at && last_update_started_at > timestamp && @@ -70,6 +71,7 @@ module Clusters ) end + # Deprecated, to be removed in %14.0 as part of https://gitlab.com/groups/gitlab-org/-/epics/4280 def patch_command(values) helm_command_module::PatchCommand.new( name: name, @@ -98,23 +100,8 @@ module Clusters files.merge('values.yaml': replaced_values) end - def generate_alert_manager_token! - unless alert_manager_token.present? - update!(alert_manager_token: generate_token) - end - end - private - def generate_token - SecureRandom.hex - end - - def disable_prometheus_integration - ::Clusters::Applications::DeactivateServiceWorker - .perform_async(cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass - end - def install_knative_metrics return [] unless cluster.application_knative_available? diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index bc80bcd0b06..e8d56072b89 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.27.0' + VERSION = '0.28.0' self.table_name = 'clusters_applications_runners' diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index a1e2aa194a0..4877ced795c 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -52,6 +52,7 @@ module Clusters has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', inverse_of: :cluster, autosave: true has_one :integration_prometheus, class_name: 'Clusters::Integrations::Prometheus', inverse_of: :cluster + has_one :integration_elastic_stack, class_name: 'Clusters::Integrations::ElasticStack', inverse_of: :cluster def self.has_one_cluster_application(name) # rubocop:disable Naming/PredicateName application = APPLICATIONS[name.to_s] @@ -104,6 +105,7 @@ module Clusters delegate :available?, to: :application_ingress, prefix: true, allow_nil: true delegate :available?, to: :application_knative, prefix: true, allow_nil: true delegate :available?, to: :application_elastic_stack, prefix: true, allow_nil: true + delegate :available?, to: :integration_elastic_stack, prefix: true, allow_nil: true delegate :external_ip, to: :application_ingress, prefix: true, allow_nil: true delegate :external_hostname, to: :application_ingress, prefix: true, allow_nil: true @@ -284,6 +286,10 @@ module Clusters integration_prometheus || build_integration_prometheus end + def find_or_build_integration_elastic_stack + integration_elastic_stack || build_integration_elastic_stack + end + def provider if gcp? provider_gcp @@ -318,6 +324,22 @@ module Clusters platform_kubernetes.kubeclient if kubernetes? end + def elastic_stack_adapter + application_elastic_stack || integration_elastic_stack + end + + def elasticsearch_client + elastic_stack_adapter&.elasticsearch_client + end + + def elastic_stack_available? + if application_elastic_stack_available? || integration_elastic_stack_available? + true + else + false + end + end + def kubernetes_namespace_for(environment, deployable: environment.last_deployable) if deployable && environment.project_id != deployable.project_id raise ArgumentError, 'environment.project_id must match deployable.project_id' @@ -470,4 +492,4 @@ module Clusters end end -Clusters::Cluster.prepend_if_ee('EE::Clusters::Cluster') +Clusters::Cluster.prepend_mod_with('Clusters::Cluster') diff --git a/app/models/clusters/concerns/application_core.rb b/app/models/clusters/concerns/application_core.rb index ad6699daa78..2e40689a650 100644 --- a/app/models/clusters/concerns/application_core.rb +++ b/app/models/clusters/concerns/application_core.rb @@ -6,6 +6,8 @@ module Clusters extend ActiveSupport::Concern included do + include ::Clusters::Concerns::KubernetesLogger + belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id validates :cluster, presence: true @@ -79,27 +81,9 @@ module Clusters # Override if your application needs any action after # being uninstalled by Helm end - - def logger - @logger ||= Gitlab::Kubernetes::Logger.build - end - - def log_exception(error, event) - logger.error({ - exception: error.class.name, - status_code: error.error_code, - cluster_id: cluster&.id, - application_id: id, - class_name: self.class.name, - event: event, - message: error.message - }) - - Gitlab::ErrorTracking.track_exception(error, cluster_id: cluster&.id, application_id: id) - end end end end end -Clusters::Concerns::ApplicationCore.prepend_if_ee('EE::Clusters::Concerns::ApplicationCore') +Clusters::Concerns::ApplicationCore.prepend_mod_with('Clusters::Concerns::ApplicationCore') diff --git a/app/models/clusters/concerns/elasticsearch_client.rb b/app/models/clusters/concerns/elasticsearch_client.rb new file mode 100644 index 00000000000..7b0b6bdae02 --- /dev/null +++ b/app/models/clusters/concerns/elasticsearch_client.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Clusters + module Concerns + module ElasticsearchClient + include ::Gitlab::Utils::StrongMemoize + + ELASTICSEARCH_PORT = 9200 + ELASTICSEARCH_NAMESPACE = 'gitlab-managed-apps' + + def elasticsearch_client(timeout: nil) + strong_memoize(:elasticsearch_client) do + kube_client = cluster&.kubeclient&.core_client + next unless kube_client + + proxy_url = kube_client.proxy_url('service', service_name, ELASTICSEARCH_PORT, ELASTICSEARCH_NAMESPACE) + + Elasticsearch::Client.new(url: proxy_url) do |faraday| + # ensures headers containing auth data are appended to original client options + faraday.headers.merge!(kube_client.headers) + # ensure TLS certs are properly verified + faraday.ssl[:verify] = kube_client.ssl_options[:verify_ssl] + faraday.ssl[:cert_store] = kube_client.ssl_options[:cert_store] + faraday.options.timeout = timeout unless timeout.nil? + end + + rescue Kubeclient::HttpError => error + # If users have mistakenly set parameters or removed the depended clusters, + # `proxy_url` could raise an exception because gitlab can not communicate with the cluster. + # We check for a nil client in downstream use and behaviour is equivalent to an empty state + log_exception(error, :failed_to_create_elasticsearch_client) + + nil + end + end + end + end +end diff --git a/app/models/clusters/concerns/kubernetes_logger.rb b/app/models/clusters/concerns/kubernetes_logger.rb new file mode 100644 index 00000000000..2eca33a7610 --- /dev/null +++ b/app/models/clusters/concerns/kubernetes_logger.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Clusters + module Concerns + module KubernetesLogger + def logger + @logger ||= Gitlab::Kubernetes::Logger.build + end + + def log_exception(error, event) + logger.error( + { + exception: error.class.name, + status_code: error.error_code, + cluster_id: cluster&.id, + application_id: id, + class_name: self.class.name, + event: event, + message: error.message + } + ) + + Gitlab::ErrorTracking.track_exception(error, cluster_id: cluster&.id, application_id: id) + end + end + end +end diff --git a/app/models/clusters/integrations/elastic_stack.rb b/app/models/clusters/integrations/elastic_stack.rb new file mode 100644 index 00000000000..565d268259a --- /dev/null +++ b/app/models/clusters/integrations/elastic_stack.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Clusters + module Integrations + class ElasticStack < ApplicationRecord + include ::Clusters::Concerns::ElasticsearchClient + include ::Clusters::Concerns::KubernetesLogger + + self.table_name = 'clusters_integration_elasticstack' + self.primary_key = :cluster_id + + belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id + + validates :cluster, presence: true + validates :enabled, inclusion: { in: [true, false] } + + def available? + enabled + end + + def service_name + chart_above_v3? ? 'elastic-stack-elasticsearch-master' : 'elastic-stack-elasticsearch-client' + end + + def chart_above_v2? + return true if chart_version.nil? + + Gem::Version.new(chart_version) >= Gem::Version.new('2.0.0') + end + + def chart_above_v3? + return true if chart_version.nil? + + Gem::Version.new(chart_version) >= Gem::Version.new('3.0.0') + end + end + end +end diff --git a/app/models/clusters/integrations/prometheus.rb b/app/models/clusters/integrations/prometheus.rb index 1496d8ff1dd..0a01ac5d1ce 100644 --- a/app/models/clusters/integrations/prometheus.rb +++ b/app/models/clusters/integrations/prometheus.rb @@ -4,6 +4,7 @@ module Clusters module Integrations class Prometheus < ApplicationRecord include ::Clusters::Concerns::PrometheusClient + include AfterCommitQueue self.table_name = 'clusters_integration_prometheus' self.primary_key = :cluster_id @@ -13,9 +14,46 @@ module Clusters validates :cluster, presence: true validates :enabled, inclusion: { in: [true, false] } + attr_encrypted :alert_manager_token, + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm' + + default_value_for(:alert_manager_token) { SecureRandom.hex } + + after_destroy do + run_after_commit do + deactivate_project_services + end + end + + after_save do + next unless enabled_before_last_save != enabled + + run_after_commit do + if enabled + activate_project_services + else + deactivate_project_services + end + end + end + def available? enabled? end + + private + + def activate_project_services + ::Clusters::Applications::ActivateServiceWorker + .perform_async(cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass + end + + def deactivate_project_services + ::Clusters::Applications::DeactivateServiceWorker + .perform_async(cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass + end end end end diff --git a/app/models/clusters/providers/aws.rb b/app/models/clusters/providers/aws.rb index bfd01775620..af2eba42721 100644 --- a/app/models/clusters/providers/aws.rb +++ b/app/models/clusters/providers/aws.rb @@ -18,7 +18,7 @@ module Clusters attr_encrypted :secret_access_key, mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base_truncated, + key: Settings.attr_encrypted_db_key_base_32, algorithm: 'aes-256-gcm' validates :role_arn, -- cgit v1.2.1