diff options
author | Shinya Maeda <shinya@gitlab.com> | 2018-01-06 01:11:06 +0900 |
---|---|---|
committer | Shinya Maeda <shinya@gitlab.com> | 2018-01-08 16:22:18 +0900 |
commit | 183dbdc8b8a68bdcbfb465abcbdae2d2292d9386 (patch) | |
tree | 514bd8fb1200a0c13661f2f3a0b90a0a969ab086 | |
parent | 1c404c91b6e0bb0fac335083eff2f286d0da6df1 (diff) | |
download | gitlab-ce-183dbdc8b8a68bdcbfb465abcbdae2d2292d9386.tar.gz |
Revert bulk_insert and bring back AR insert(one by one)
-rw-r--r-- | db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb | 78 | ||||
-rw-r--r-- | spec/migrations/migrate_kubernetes_service_to_new_clusters_architectures_spec.rb | 40 |
2 files changed, 54 insertions, 64 deletions
diff --git a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb index 2808c8e0222..27d56fe5b42 100644 --- a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb +++ b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb @@ -1,7 +1,11 @@ class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + DOWNTIME = false DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME = 'KubernetesService'.freeze + disable_ddl_transaction! + class Project < ActiveRecord::Base self.table_name = 'projects' @@ -58,14 +62,16 @@ class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migrati joins('LEFT JOIN projects ON projects.id = services.project_id') .joins('LEFT JOIN cluster_projects ON cluster_projects.project_id = projects.id') .joins('LEFT JOIN cluster_platforms_kubernetes ON cluster_platforms_kubernetes.cluster_id = cluster_projects.cluster_id') - .where(category: 'deployment') - .where(type: 'KubernetesService') - .where(template: false) + .where(category: 'deployment', type: 'KubernetesService', template: false) .where("services.properties LIKE '%api_url%'") .where("(services.properties NOT LIKE CONCAT('%', cluster_platforms_kubernetes.api_url, '%')) OR cluster_platforms_kubernetes.api_url IS NULL") .group(:id) .order(id: :asc) end + + scope :kubernetes_service_without_template, -> do + where(category: 'deployment', type: 'KubernetesService', template: false) + end end def find_dedicated_environement_scope(project) @@ -86,65 +92,33 @@ class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migrati end def up - MigrateKubernetesServiceToNewClustersArchitectures::Service - .unmanaged_kubernetes_service.each_batch(of: 100) do |kubernetes_services| - - rows_for_clusters = kubernetes_services.map do |kubernetes_service| - { + ActiveRecord::Base.transaction do + MigrateKubernetesServiceToNewClustersArchitectures::Service + .unmanaged_kubernetes_service.find_each(batch_size: 1) do |kubernetes_service| + MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create( enabled: kubernetes_service.active, user_id: nil, # KubernetesService doesn't have name: DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME, provider_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.provider_types[:user], platform_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.platform_types[:kubernetes], + projects: [kubernetes_service.project.becomes(MigrateKubernetesServiceToNewClustersArchitectures::Project)], environment_scope: find_dedicated_environement_scope(kubernetes_service.project), - created_at: Gitlab::Database.sanitize_timestamp(kubernetes_service.created_at), - updated_at: Gitlab::Database.sanitize_timestamp(kubernetes_service.updated_at) - } - end - - inserted_cluster_ids = Gitlab::Database.bulk_insert('clusters', rows_for_clusters, return_ids: true) - - rows_for_cluster_platforms_kubernetes = kubernetes_services.each_with_index.map do |kubernetes_service, i| - - # Create PlatformsKubernetes instance for generating an encrypted token - platforms_kubernetes = - MigrateKubernetesServiceToNewClustersArchitectures::PlatformsKubernetes - .new(token: kubernetes_service.token) - - { - cluster_id: inserted_cluster_ids[i], - api_url: kubernetes_service.api_url, - ca_cert: kubernetes_service.ca_pem, - namespace: kubernetes_service.namespace, - username: nil, # KubernetesService doesn't have - encrypted_password: nil, # KubernetesService doesn't have - encrypted_password_iv: nil, # KubernetesService doesn't have - encrypted_token: platforms_kubernetes.encrypted_token, # encrypted_token and encrypted_token_iv - encrypted_token_iv: platforms_kubernetes.encrypted_token_iv, # encrypted_token and encrypted_token_iv - created_at: Gitlab::Database.sanitize_timestamp(kubernetes_service.created_at), - updated_at: Gitlab::Database.sanitize_timestamp(kubernetes_service.updated_at) - } - end - - Gitlab::Database.bulk_insert('cluster_platforms_kubernetes', rows_for_cluster_platforms_kubernetes) - - rows_for_cluster_projects = kubernetes_services.each_with_index.map do |kubernetes_service, i| - { - cluster_id: inserted_cluster_ids[i], - project_id: kubernetes_service.project_id, - created_at: Gitlab::Database.sanitize_timestamp(kubernetes_service.created_at), - updated_at: Gitlab::Database.sanitize_timestamp(kubernetes_service.updated_at) - } + platform_kubernetes_attributes: { + api_url: kubernetes_service.api_url, + ca_cert: kubernetes_service.ca_pem, + namespace: kubernetes_service.namespace, + username: nil, # KubernetesService doesn't have + encrypted_password: nil, # KubernetesService doesn't have + encrypted_password_iv: nil, # KubernetesService doesn't have + token: kubernetes_service.token # encrypted_token and encrypted_token_iv + } ) end - - Gitlab::Database.bulk_insert('cluster_projects', rows_for_cluster_projects) end MigrateKubernetesServiceToNewClustersArchitectures::Service - .where(category: 'deployment', type: 'KubernetesService', template: false) - .each_batch(of: 100) do |batch| - batch.update_all(active: false) - end + .kubernetes_service_without_template.each_batch(of: 100) do |kubernetes_service| + kubernetes_service.update_all(active: false) + end end def down diff --git a/spec/migrations/migrate_kubernetes_service_to_new_clusters_architectures_spec.rb b/spec/migrations/migrate_kubernetes_service_to_new_clusters_architectures_spec.rb index 5f135ad274e..85ed1328211 100644 --- a/spec/migrations/migrate_kubernetes_service_to_new_clusters_architectures_spec.rb +++ b/spec/migrations/migrate_kubernetes_service_to_new_clusters_architectures_spec.rb @@ -38,18 +38,6 @@ describe MigrateKubernetesServiceToNewClustersArchitectures, :migration do it_behaves_like 'KubernetesService migration' end - - context 'when KubernetesService is not active' do - let(:active) { false } - - # Platforms::Kubernetes validates `token` reagdless of the activeness, - # whereas KubernetesService validates `token` if only it's activated - # However, in this migration file, there are no validations because of the migration specific model class - # therefore, Validation Error will not happen in this case and just migrate data - let(:token) { '' } - - it_behaves_like 'KubernetesService migration' - end end context 'when unique KubernetesService spawned from Service Template' do @@ -186,6 +174,34 @@ describe MigrateKubernetesServiceToNewClustersArchitectures, :migration do end end + # Platforms::Kubernetes validates `token` reagdless of the activeness, + # whereas KubernetesService validates `token` if only it's activated + # However, in this migration file, there are no validations because of the re-defined model class + # therefore, we should safely add this raw to Platform::Kubernetes + context 'when KubernetesService has empty token' do + let(:project) { create(:project) } + + before do + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO services (project_id, active, category, type, properties) + VALUES (#{project.id}, false, 'deployment', 'KubernetesService', '{"namespace":"prod","api_url":"http://111.111.111.111","ca_pem":"a","token":""}'); + SQL + end + + it 'does not migrate the KubernetesService and disables the kubernetes_service' do + expect { migrate! }.to change { Clusters::Cluster.count }.by(1) + + project.clusters.last.tap do |cluster| + expect(cluster.environment_scope).to eq('*') + expect(cluster.platform_kubernetes.namespace).to eq('prod') + expect(cluster.platform_kubernetes.api_url).to eq('http://111.111.111.111') + expect(cluster.platform_kubernetes.ca_pem).to eq('a') + expect(cluster.platform_kubernetes.token).to be_empty + expect(project.kubernetes_service).not_to be_active + end + end + end + context 'when KubernetesService does not exist' do let!(:project) { create(:project) } |