Merge branch '40418-migrate-existing-data-from-kubernetesservice-to-clusters-platforms-kubernetes' into 'master'

Migrate existing data from KubernetesService to Clusters::Platforms::Kubernetes

Closes #40418

See merge request gitlab-org/gitlab-ce!15589

(cherry picked from commit 4b92efd90cedaa0aff218d11fdce279701128bea)

5b2ca1c6 Migrate existing data from KubernetesService to Clusters::Platforms::Kubernetes
e4745492 Add test. Disable KubernetesService when migrated
b9fbfe5a Fix unmanaged_kubernetes_service scope for multiple clusters
40c6af54 Fix migration file typos and reorder Table definition
27111e29 Restructure spec
4dc14576 Fix comments
8e6ffe35 Fix test
7eeada80 Add env_scope tests
f083739e Add logic to swtich environment_scope by the situation
9b7719b6 Use explicit namespace for avoiding reference from application code
665972e2 Avoid quotes in ActiveRecord query
c8059881 Opitmize migration process by using both unmanaged_kubernetes_service and…
b8a275d3 Use bulk_insert instead of AR create
acfb8464 Fix static anylysy
8bc3221f Fix query to look for proper unmanaged kubernetes service
2d3c7d29 Use batch update for Service deactivation
1c404c91 Add a new test for emptified params
183dbdc8 Revert bulk_insert and bring back AR insert(one by one)
54d20d1b Add changelog
290c2248 Fix change log
58d074e0 Fix StaticSnalysys
df658c7b Disable STI of ActiveRecord. Refactoring specs.
c425ff75 Fix static analysys
67327952 Add memoization for properties

1 files changed, 151 insertions, 0 deletions

diff --git a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb
new file mode 100644
index 00000000000..11b581e4b57
--- /dev/null
+++ b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb
@@ -0,0 +1,151 @@
+class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME = 'KubernetesService'.freeze
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject'
+    has_many :clusters, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+    has_many :services, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service'
+    has_one :kubernetes_service, -> { where(category: 'deployment', type: 'KubernetesService') }, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service', inverse_of: :project, foreign_key: :project_id
+  end
+
+  class Cluster < ActiveRecord::Base
+    self.table_name = 'clusters'
+
+    has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject'
+    has_many :projects, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project'
+    has_one :platform_kubernetes, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::PlatformsKubernetes'
+
+    accepts_nested_attributes_for :platform_kubernetes
+
+    enum platform_type: {
+      kubernetes: 1
+    }
+
+    enum provider_type: {
+      user: 0,
+      gcp: 1
+    }
+  end
+
+  class ClustersProject < ActiveRecord::Base
+    self.table_name = 'cluster_projects'
+
+    belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+    belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project'
+  end
+
+  class PlatformsKubernetes < ActiveRecord::Base
+    self.table_name = 'cluster_platforms_kubernetes'
+
+    belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+
+    attr_encrypted :token,
+      mode: :per_attribute_iv,
+      key: Gitlab::Application.secrets.db_key_base,
+      algorithm: 'aes-256-cbc'
+  end
+
+  class Service < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'services'
+    self.inheritance_column = :_type_disabled # Disable STI, otherwise KubernetesModel will be looked up
+
+    belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project', foreign_key: :project_id
+
+    scope :unmanaged_kubernetes_service, -> do
+      joins('LEFT JOIN projects ON projects.id = services.project_id')
+      .joins('LEFT JOIN cluster_projects ON cluster_projects.project_id = projects.id')
+      .joins('LEFT JOIN cluster_platforms_kubernetes ON cluster_platforms_kubernetes.cluster_id = cluster_projects.cluster_id')
+      .where(category: 'deployment', type: 'KubernetesService', template: false)
+      .where("services.properties LIKE '%api_url%'")
+      .where("(services.properties NOT LIKE CONCAT('%', cluster_platforms_kubernetes.api_url, '%')) OR cluster_platforms_kubernetes.api_url IS NULL")
+      .group(:id)
+      .order(id: :asc)
+    end
+
+    scope :kubernetes_service_without_template, -> do
+      where(category: 'deployment', type: 'KubernetesService', template: false)
+    end
+
+    def api_url
+      parsed_properties['api_url']
+    end
+
+    def ca_pem
+      parsed_properties['ca_pem']
+    end
+
+    def namespace
+      parsed_properties['namespace']
+    end
+
+    def token
+      parsed_properties['token']
+    end
+
+    private
+
+    def parsed_properties
+      @parsed_properties ||= JSON.parse(self.properties)
+    end
+  end
+
+  def find_dedicated_environement_scope(project)
+    environment_scopes = project.clusters.map(&:environment_scope)
+
+    return '*' if environment_scopes.exclude?('*') # KubernetesService should be added as a default cluster (environment_scope: '*') at first place
+    return 'migrated/*' if environment_scopes.exclude?('migrated/*') # If it's conflicted, the KubernetesService added as a migrated cluster
+
+    unique_iid = 0
+
+    # If it's still conflicted, finding an unique environment scope incrementaly
+    loop do
+      candidate = "migrated#{unique_iid}/*"
+      return candidate if environment_scopes.exclude?(candidate)
+
+      unique_iid += 1
+    end
+  end
+
+  def up
+    ActiveRecord::Base.transaction do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service
+        .unmanaged_kubernetes_service.find_each(batch_size: 1) do |kubernetes_service|
+        MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create(
+          enabled: kubernetes_service.active,
+          user_id: nil, # KubernetesService doesn't have
+          name: DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME,
+          provider_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.provider_types[:user],
+          platform_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.platform_types[:kubernetes],
+          projects: [kubernetes_service.project],
+          environment_scope: find_dedicated_environement_scope(kubernetes_service.project),
+          platform_kubernetes_attributes: {
+            api_url: kubernetes_service.api_url,
+            ca_cert: kubernetes_service.ca_pem,
+            namespace: kubernetes_service.namespace,
+            username: nil, # KubernetesService doesn't have
+            encrypted_password: nil, # KubernetesService doesn't have
+            encrypted_password_iv: nil, # KubernetesService doesn't have
+            token: kubernetes_service.token # encrypted_token and encrypted_token_iv
+          } )
+      end
+    end
+
+    MigrateKubernetesServiceToNewClustersArchitectures::Service
+      .kubernetes_service_without_template.each_batch(of: 100) do |kubernetes_service|
+      kubernetes_service.update_all(active: false)
+    end
+  end
+
+  def down
+    # noop
+  end
+end