Add latest changes from gitlab-org/gitlab@12-6-stable-ee

8 files changed, 106 insertions, 42 deletions

diff --git a/app/models/clusters/applications/elastic_stack.rb b/app/models/clusters/applications/elastic_stack.rb
index 8589f8c00cb..9854ad2ea3e 100644
--- a/app/models/clusters/applications/elastic_stack.rb
+++ b/app/models/clusters/applications/elastic_stack.rb
@@ -71,6 +71,8 @@ module Clusters
           # `proxy_url` could raise an exception because gitlab can not communicate with the cluster.
           # We check for a nil client in downstream use and behaviour is equivalent to an empty state
           log_exception(error, :failed_to_create_elasticsearch_client)
+
+          nil
         end
       end
 
diff --git a/app/models/clusters/applications/knative.rb b/app/models/clusters/applications/knative.rb
index 1093efee85a..387503bee54 100644
--- a/app/models/clusters/applications/knative.rb
+++ b/app/models/clusters/applications/knative.rb
@@ -3,25 +3,27 @@
 module Clusters
   module Applications
     class Knative < ApplicationRecord
-      VERSION = '0.7.0'
+      VERSION = '0.9.0'
       REPOSITORY = 'https://storage.googleapis.com/triggermesh-charts'
       METRICS_CONFIG = 'https://storage.googleapis.com/triggermesh-charts/istio-metrics.yaml'
       FETCH_IP_ADDRESS_DELAY = 30.seconds
-      API_RESOURCES_PATH = 'config/knative/api_resources.yml'
+      API_GROUPS_PATH = 'config/knative/api_groups.yml'
 
       self.table_name = 'clusters_applications_knative'
 
+      has_one :serverless_domain_cluster, class_name: 'Serverless::DomainCluster', foreign_key: 'clusters_applications_knative_id', inverse_of: :knative
+
       include ::Clusters::Concerns::ApplicationCore
       include ::Clusters::Concerns::ApplicationStatus
       include ::Clusters::Concerns::ApplicationVersion
       include ::Clusters::Concerns::ApplicationData
       include AfterCommitQueue
 
+      alias_method :original_set_initial_status, :set_initial_status
       def set_initial_status
-        return unless not_installable?
-        return unless verify_cluster?
+        return unless cluster&.platform_kubernetes_rbac?
 
-        self.status = status_states[:installable]
+        original_set_initial_status
       end
 
       state_machine :status do
@@ -109,15 +111,15 @@ module Clusters
       end
 
       def delete_knative_and_istio_crds
-        api_resources.map do |crd|
-          Gitlab::Kubernetes::KubectlCmd.delete("--ignore-not-found", "crd", "#{crd}")
+        api_groups.map do |group|
+          Gitlab::Kubernetes::KubectlCmd.delete_crds_from_group(group)
         end
       end
 
       # returns an array of CRDs to be postdelete since helm does not
       # manage the CRDs it creates.
-      def api_resources
-        @api_resources ||= YAML.safe_load(File.read(Rails.root.join(API_RESOURCES_PATH)))
+      def api_groups
+        @api_groups ||= YAML.safe_load(File.read(Rails.root.join(API_GROUPS_PATH)))
       end
 
       def install_knative_metrics
@@ -131,10 +133,6 @@ module Clusters
 
         [Gitlab::Kubernetes::KubectlCmd.delete("--ignore-not-found", "-f", METRICS_CONFIG)]
       end
-
-      def verify_cluster?
-        cluster&.application_helm_available? && cluster&.platform_kubernetes_rbac?
-      end
     end
   end
 end
diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb
index 5e7fdd55cb6..4ac33d4e3be 100644
--- a/app/models/clusters/applications/prometheus.rb
+++ b/app/models/clusters/applications/prometheus.rb
@@ -13,15 +13,21 @@ module Clusters
       include ::Clusters::Concerns::ApplicationStatus
       include ::Clusters::Concerns::ApplicationVersion
       include ::Clusters::Concerns::ApplicationData
+      include AfterCommitQueue
 
       default_value_for :version, VERSION
 
-      after_destroy :disable_prometheus_integration
+      after_destroy do
+        run_after_commit do
+          disable_prometheus_integration
+        end
+      end
 
       state_machine :status do
         after_transition any => [:installed] do |application|
-          application.cluster.projects.each do |project|
-            project.find_or_initialize_service('prometheus').update!(active: true)
+          application.run_after_commit do
+            Clusters::Applications::ActivateServiceWorker
+              .perform_async(application.cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass
           end
         end
       end
@@ -49,10 +55,10 @@ module Clusters
         )
       end
 
-      def upgrade_command(values)
-        ::Gitlab::Kubernetes::Helm::InstallCommand.new(
+      def patch_command(values)
+        ::Gitlab::Kubernetes::Helm::PatchCommand.new(
           name: name,
-          version: VERSION,
+          version: version,
           rbac: cluster.platform_kubernetes_rbac?,
           chart: chart,
           files: files_with_replaced_values(values)
@@ -84,19 +90,22 @@ module Clusters
         # ensures headers containing auth data are appended to original k8s client options
         options = kube_client.rest_client.options.merge(headers: kube_client.headers)
         Gitlab::PrometheusClient.new(proxy_url, options)
-      rescue Kubeclient::HttpError
+      rescue Kubeclient::HttpError, Errno::ECONNRESET, Errno::ECONNREFUSED
         # If users have mistakenly set parameters or removed the depended clusters,
         # `proxy_url` could raise an exception because gitlab can not communicate with the cluster.
         # Since `PrometheusAdapter#can_query?` is eargely loaded on environement pages in gitlab,
         # we need to silence the exceptions
       end
 
+      def configured?
+        kube_client.present? && available?
+      end
+
       private
 
       def disable_prometheus_integration
-        cluster.projects.each do |project|
-          project.prometheus_service&.update!(active: false)
-        end
+        ::Clusters::Applications::DeactivateServiceWorker
+          .perform_async(cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass
       end
 
       def kube_client
diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb
index 37ba8a7c97e..fd05fd6bab9 100644
--- a/app/models/clusters/applications/runner.rb
+++ b/app/models/clusters/applications/runner.rb
@@ -3,7 +3,7 @@
 module Clusters
   module Applications
     class Runner < ApplicationRecord
-      VERSION = '0.10.1'
+      VERSION = '0.11.0'
 
       self.table_name = 'clusters_applications_runners'
 
diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb
index f522f3f2fdb..d2eee78f3df 100644
--- a/app/models/clusters/cluster.rb
+++ b/app/models/clusters/cluster.rb
@@ -23,6 +23,7 @@ module Clusters
     }.freeze
     DEFAULT_ENVIRONMENT = '*'
     KUBE_INGRESS_BASE_DOMAIN = 'KUBE_INGRESS_BASE_DOMAIN'
+    APPLICATIONS_ASSOCIATIONS = APPLICATIONS.values.map(&:association_name).freeze
 
     belongs_to :user
     belongs_to :management_project, class_name: '::Project', optional: true
@@ -33,6 +34,7 @@ module Clusters
 
     has_many :cluster_groups, class_name: 'Clusters::Group'
     has_many :groups, through: :cluster_groups, class_name: '::Group'
+    has_many :groups_projects, through: :groups, source: :projects, class_name: '::Project'
 
     # we force autosave to happen when we save `Cluster` model
     has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true
@@ -117,7 +119,7 @@ module Clusters
     scope :aws_installed, -> { aws_provided.joins(:provider_aws).merge(Clusters::Providers::Aws.with_status(:created)) }
 
     scope :managed, -> { where(managed: true) }
-
+    scope :with_persisted_applications, -> { eager_load(*APPLICATIONS_ASSOCIATIONS) }
     scope :default_environment, -> { where(environment_scope: DEFAULT_ENVIRONMENT) }
 
     scope :for_project_namespace, -> (namespace_id) { joins(:projects).where(projects: { namespace_id: namespace_id }) }
@@ -176,6 +178,13 @@ module Clusters
       end
     end
 
+    def all_projects
+      return projects if project_type?
+      return groups_projects if group_type?
+
+      ::Project.all
+    end
+
     def status_name
       return cleanup_status_name if cleanup_errored?
       return :cleanup_ongoing unless cleanup_not_started?
@@ -195,9 +204,13 @@ module Clusters
       { connection_status: retrieve_connection_status }
     end
 
+    def persisted_applications
+      APPLICATIONS_ASSOCIATIONS.map(&method(:public_send)).compact
+    end
+
     def applications
-      APPLICATIONS.values.map do |application_class|
-        public_send(application_class.association_name) || public_send("build_#{application_class.association_name}") # rubocop:disable GitlabSecurity/PublicSend
+      APPLICATIONS_ASSOCIATIONS.map do |association_name|
+        public_send(association_name) || public_send("build_#{association_name}") # rubocop:disable GitlabSecurity/PublicSend
       end
     end
 
@@ -236,14 +249,9 @@ module Clusters
     end
 
     def kubernetes_namespace_for(environment)
-      project = environment.project
-      persisted_namespace = Clusters::KubernetesNamespaceFinder.new(
-        self,
-        project: project,
-        environment_name: environment.name
-      ).execute
-
-      persisted_namespace&.namespace || Gitlab::Kubernetes::DefaultNamespace.new(self, project: project).from_environment_slug(environment.slug)
+      managed_namespace(environment) ||
+        ci_configured_namespace(environment) ||
+        default_namespace(environment)
     end
 
     def allow_user_defined_namespace?
@@ -262,6 +270,25 @@ module Clusters
       end
     end
 
+    def delete_cached_resources!
+      kubernetes_namespaces.delete_all(:delete_all)
+    end
+
+    def clusterable
+      return unless cluster_type
+
+      case cluster_type
+      when 'project_type'
+        project
+      when 'group_type'
+        group
+      when 'instance_type'
+        instance
+      else
+        raise NotImplementedError
+      end
+    end
+
     private
 
     def unique_management_project_environment_scope
@@ -276,6 +303,25 @@ module Clusters
       end
     end
 
+    def managed_namespace(environment)
+      Clusters::KubernetesNamespaceFinder.new(
+        self,
+        project: environment.project,
+        environment_name: environment.name
+      ).execute&.namespace
+    end
+
+    def ci_configured_namespace(environment)
+      environment.last_deployable&.expanded_kubernetes_namespace
+    end
+
+    def default_namespace(environment)
+      Gitlab::Kubernetes::DefaultNamespace.new(
+        self,
+        project: environment.project
+      ).from_environment_slug(environment.slug)
+    end
+
     def instance_domain
       @instance_domain ||= Gitlab::CurrentSettings.auto_devops_domain
     end
@@ -289,7 +335,7 @@ module Clusters
     rescue Kubeclient::HttpError => e
       kubeclient_error_status(e.message)
     rescue => e
-      Gitlab::Sentry.track_acceptable_exception(e, extra: { cluster_id: id })
+      Gitlab::ErrorTracking.track_exception(e, cluster_id: id)
 
       :unknown_failure
     else
diff --git a/app/models/clusters/concerns/application_core.rb b/app/models/clusters/concerns/application_core.rb
index 21b98534808..f6431f5bac3 100644
--- a/app/models/clusters/concerns/application_core.rb
+++ b/app/models/clusters/concerns/application_core.rb
@@ -76,7 +76,7 @@ module Clusters
             message: error.message
           })
 
-          Gitlab::Sentry.track_acceptable_exception(error, extra: { cluster_id: cluster&.id, application_id: id })
+          Gitlab::ErrorTracking.track_exception(error, cluster_id: cluster&.id, application_id: id)
         end
       end
     end
diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb
index 314ef78757d..ae720065387 100644
--- a/app/models/clusters/platforms/kubernetes.rb
+++ b/app/models/clusters/platforms/kubernetes.rb
@@ -63,7 +63,7 @@ module Clusters
 
       default_value_for :authorization_type, :rbac
 
-      def predefined_variables(project:, environment_name:)
+      def predefined_variables(project:, environment_name:, kubernetes_namespace: nil)
         Gitlab::Ci::Variables::Collection.new.tap do |variables|
           variables.append(key: 'KUBE_URL', value: api_url)
 
@@ -74,15 +74,15 @@ module Clusters
           end
 
           if !cluster.managed? || cluster.management_project == project
-            namespace = Gitlab::Kubernetes::DefaultNamespace.new(cluster, project: project).from_environment_name(environment_name)
+            namespace = kubernetes_namespace || default_namespace(project, environment_name: environment_name)
 
             variables
               .append(key: 'KUBE_TOKEN', value: token, public: false, masked: true)
               .append(key: 'KUBE_NAMESPACE', value: namespace)
               .append(key: 'KUBECONFIG', value: kubeconfig(namespace), public: false, file: true)
 
-          elsif kubernetes_namespace = find_persisted_namespace(project, environment_name: environment_name)
-            variables.concat(kubernetes_namespace.predefined_variables)
+          elsif persisted_namespace = find_persisted_namespace(project, environment_name: environment_name)
+            variables.concat(persisted_namespace.predefined_variables)
           end
 
           variables.concat(cluster.predefined_variables)
@@ -107,6 +107,13 @@ module Clusters
 
       private
 
+      def default_namespace(project, environment_name:)
+        Gitlab::Kubernetes::DefaultNamespace.new(
+          cluster,
+          project: project
+        ).from_environment_name(environment_name)
+      end
+
       def find_persisted_namespace(project, environment_name:)
         Clusters::KubernetesNamespaceFinder.new(
           cluster,
diff --git a/app/models/clusters/providers/aws.rb b/app/models/clusters/providers/aws.rb
index 78eb75ddcc0..faf587fb83d 100644
--- a/app/models/clusters/providers/aws.rb
+++ b/app/models/clusters/providers/aws.rb
@@ -8,9 +8,11 @@ module Clusters
 
       self.table_name = 'cluster_providers_aws'
 
+      DEFAULT_REGION = 'us-east-1'
+
       belongs_to :cluster, inverse_of: :provider_aws, class_name: 'Clusters::Cluster'
 
-      default_value_for :region, 'us-east-1'
+      default_value_for :region, DEFAULT_REGION
       default_value_for :num_nodes, 3
       default_value_for :instance_type, 'm5.large'