Add latest changes from gitlab-org/gitlab@13-0-stable-ee

6 files changed, 70 insertions, 12 deletions

diff --git a/app/models/clusters/applications/elastic_stack.rb b/app/models/clusters/applications/elastic_stack.rb
index afdc1c91c69..0d029aabc3b 100644
--- a/app/models/clusters/applications/elastic_stack.rb
+++ b/app/models/clusters/applications/elastic_stack.rb
@@ -3,7 +3,7 @@
 module Clusters
   module Applications
     class ElasticStack < ApplicationRecord
-      VERSION = '1.9.0'
+      VERSION = '3.0.0'
 
       ELASTICSEARCH_PORT = 9200
 
@@ -18,7 +18,11 @@ module Clusters
       default_value_for :version, VERSION
 
       def chart
-        'stable/elastic-stack'
+        'elastic-stack/elastic-stack'
+      end
+
+      def repository
+        'https://charts.gitlab.io'
       end
 
       def install_command
@@ -27,7 +31,9 @@ module Clusters
           version: VERSION,
           rbac: cluster.platform_kubernetes_rbac?,
           chart: chart,
+          repository: repository,
           files: files,
+          preinstall: migrate_to_3_script,
           postinstall: post_install_script
         )
       end
@@ -49,7 +55,7 @@ module Clusters
         strong_memoize(:elasticsearch_client) do
           next unless kube_client
 
-          proxy_url = kube_client.proxy_url('service', 'elastic-stack-elasticsearch-client', ::Clusters::Applications::ElasticStack::ELASTICSEARCH_PORT, Gitlab::Kubernetes::Helm::NAMESPACE)
+          proxy_url = kube_client.proxy_url('service', service_name, ::Clusters::Applications::ElasticStack::ELASTICSEARCH_PORT, Gitlab::Kubernetes::Helm::NAMESPACE)
 
           Elasticsearch::Client.new(url: proxy_url) do |faraday|
             # ensures headers containing auth data are appended to original client options
@@ -69,23 +75,54 @@ module Clusters
         end
       end
 
+      def chart_above_v2?
+        Gem::Version.new(version) >= Gem::Version.new('2.0.0')
+      end
+
+      def chart_above_v3?
+        Gem::Version.new(version) >= Gem::Version.new('3.0.0')
+      end
+
       private
 
+      def service_name
+        chart_above_v3? ? 'elastic-stack-elasticsearch-master' : 'elastic-stack-elasticsearch-client'
+      end
+
+      def pvc_selector
+        chart_above_v3? ? "app=elastic-stack-elasticsearch-master" : "release=elastic-stack"
+      end
+
       def post_install_script
         [
-          "timeout -t60 sh /data/helm/elastic-stack/config/wait-for-elasticsearch.sh http://elastic-stack-elasticsearch-client:9200"
+          "timeout -t60 sh /data/helm/elastic-stack/config/wait-for-elasticsearch.sh http://elastic-stack-elasticsearch-master:9200"
         ]
       end
 
       def post_delete_script
         [
-          Gitlab::Kubernetes::KubectlCmd.delete("pvc", "--selector", "release=elastic-stack")
+          Gitlab::Kubernetes::KubectlCmd.delete("pvc", "--selector", pvc_selector, "--namespace", Gitlab::Kubernetes::Helm::NAMESPACE)
         ]
       end
 
       def kube_client
         cluster&.kubeclient&.core_client
       end
+
+      def migrate_to_3_script
+        return [] if !updating? || chart_above_v3?
+
+        # Chart version 3.0.0 moves to our own chart at https://gitlab.com/gitlab-org/charts/elastic-stack
+        # and is not compatible with pre-existing resources. We first remove them.
+        [
+          Gitlab::Kubernetes::Helm::DeleteCommand.new(
+            name: 'elastic-stack',
+            rbac: cluster.platform_kubernetes_rbac?,
+            files: files
+          ).delete_command,
+          Gitlab::Kubernetes::KubectlCmd.delete("pvc", "--selector", "release=elastic-stack", "--namespace", Gitlab::Kubernetes::Helm::NAMESPACE)
+        ]
+      end
     end
   end
 end
diff --git a/app/models/clusters/applications/fluentd.rb b/app/models/clusters/applications/fluentd.rb
index a33b1e39ace..3fd6e870edc 100644
--- a/app/models/clusters/applications/fluentd.rb
+++ b/app/models/clusters/applications/fluentd.rb
@@ -4,6 +4,7 @@ module Clusters
   module Applications
     class Fluentd < ApplicationRecord
       VERSION = '2.4.0'
+      CILIUM_CONTAINER_NAME = 'cilium-monitor'
 
       self.table_name = 'clusters_applications_fluentd'
 
@@ -18,6 +19,8 @@ module Clusters
 
       enum protocol: { tcp: 0, udp: 1 }
 
+      validate :has_at_least_one_log_enabled?
+
       def chart
         'stable/fluentd'
       end
@@ -39,6 +42,12 @@ module Clusters
 
       private
 
+      def has_at_least_one_log_enabled?
+        if !waf_log_enabled && !cilium_log_enabled
+          errors.add(:base, _("At least one logging option is required to be enabled"))
+        end
+      end
+
       def content_values
         YAML.load_file(chart_values_file).deep_merge!(specification)
       end
@@ -62,7 +71,7 @@ module Clusters
           program fluentd
           hostname ${kubernetes_host}
           protocol #{protocol}
-          packet_size 65535
+          packet_size 131072
           <buffer kubernetes_host>
           </buffer>
           <format>
@@ -85,7 +94,7 @@ module Clusters
         <source>
           @type tail
           @id in_tail_container_logs
-          path /var/log/containers/*#{Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log
+          path #{path_to_logs}
           pos_file /var/log/fluentd-containers.log.pos
           tag kubernetes.*
           read_from_head true
@@ -96,6 +105,13 @@ module Clusters
         </source>
         EOF
       end
+
+      def path_to_logs
+        path = []
+        path << "/var/log/containers/*#{Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log" if waf_log_enabled
+        path << "/var/log/containers/*#{CILIUM_CONTAINER_NAME}*.log" if cilium_log_enabled
+        path.join(',')
+      end
     end
   end
 end
diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb
index 5985e08d73e..dd354198910 100644
--- a/app/models/clusters/applications/ingress.rb
+++ b/app/models/clusters/applications/ingress.rb
@@ -17,6 +17,7 @@ module Clusters
       include ::Clusters::Concerns::ApplicationVersion
       include ::Clusters::Concerns::ApplicationData
       include AfterCommitQueue
+      include UsageStatistics
 
       default_value_for :ingress_type, :nginx
       default_value_for :modsecurity_enabled, true
@@ -29,6 +30,10 @@ module Clusters
 
       enum modsecurity_mode: { logging: 0, blocking: 1 }
 
+      scope :modsecurity_not_installed, -> { where(modsecurity_enabled: nil) }
+      scope :modsecurity_enabled, -> { where(modsecurity_enabled: true) }
+      scope :modsecurity_disabled, -> { where(modsecurity_enabled: false) }
+
       FETCH_IP_ADDRESS_DELAY = 30.seconds
 
       state_machine :status do
@@ -98,7 +103,7 @@ module Clusters
                 "args" => [
                   "/bin/sh",
                   "-c",
-                  "tail -f /var/log/modsec/audit.log"
+                  "tail -F /var/log/modsec/audit.log"
                 ],
                 "volumeMounts" => [
                   {
diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb
index 42fa4a6f179..056ea355de6 100644
--- a/app/models/clusters/applications/jupyter.rb
+++ b/app/models/clusters/applications/jupyter.rb
@@ -5,7 +5,7 @@ require 'securerandom'
 module Clusters
   module Applications
     class Jupyter < ApplicationRecord
-      VERSION = '0.9.0-beta.2'
+      VERSION = '0.9.0'
 
       self.table_name = 'clusters_applications_jupyter'
 
diff --git a/app/models/clusters/applications/knative.rb b/app/models/clusters/applications/knative.rb
index 1f90318f845..3047da12dd9 100644
--- a/app/models/clusters/applications/knative.rb
+++ b/app/models/clusters/applications/knative.rb
@@ -4,8 +4,8 @@ module Clusters
   module Applications
     class Knative < ApplicationRecord
       VERSION = '0.9.0'
-      REPOSITORY = 'https://storage.googleapis.com/triggermesh-charts'
-      METRICS_CONFIG = 'https://storage.googleapis.com/triggermesh-charts/istio-metrics.yaml'
+      REPOSITORY = 'https://charts.gitlab.io'
+      METRICS_CONFIG = 'https://gitlab.com/gitlab-org/charts/knative/-/raw/v0.9.0/vendor/istio-metrics.yml'
       FETCH_IP_ADDRESS_DELAY = 30.seconds
       API_GROUPS_PATH = 'config/knative/api_groups.yml'
 
diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb
index 7d67e258991..a861126908f 100644
--- a/app/models/clusters/applications/runner.rb
+++ b/app/models/clusters/applications/runner.rb
@@ -3,7 +3,7 @@
 module Clusters
   module Applications
     class Runner < ApplicationRecord
-      VERSION = '0.15.0'
+      VERSION = '0.16.1'
 
       self.table_name = 'clusters_applications_runners'