From cdc6050d574000fefdb4fbfc315496b66cbe4e68 Mon Sep 17 00:00:00 2001 From: Peter Leitzen Date: Thu, 18 Jul 2019 13:45:32 +0200 Subject: Rename CloudProvider -> ClusterProvider --- qa/qa.rb | 10 +-- qa/qa/service/cloud_provider/base.rb | 41 --------- qa/qa/service/cloud_provider/gcloud.rb | 91 ------------------- qa/qa/service/cloud_provider/k3d.rb | 137 ----------------------------- qa/qa/service/cloud_provider/minikube.rb | 26 ------ qa/qa/service/cluster_provider/base.rb | 41 +++++++++ qa/qa/service/cluster_provider/gcloud.rb | 91 +++++++++++++++++++ qa/qa/service/cluster_provider/k3d.rb | 137 +++++++++++++++++++++++++++++ qa/qa/service/cluster_provider/minikube.rb | 26 ++++++ qa/qa/service/kubernetes_cluster.rb | 6 +- 10 files changed, 303 insertions(+), 303 deletions(-) delete mode 100644 qa/qa/service/cloud_provider/base.rb delete mode 100644 qa/qa/service/cloud_provider/gcloud.rb delete mode 100644 qa/qa/service/cloud_provider/k3d.rb delete mode 100644 qa/qa/service/cloud_provider/minikube.rb create mode 100644 qa/qa/service/cluster_provider/base.rb create mode 100644 qa/qa/service/cluster_provider/gcloud.rb create mode 100644 qa/qa/service/cluster_provider/k3d.rb create mode 100644 qa/qa/service/cluster_provider/minikube.rb diff --git a/qa/qa.rb b/qa/qa.rb index dc0360b3345..a760f72f70d 100644 --- a/qa/qa.rb +++ b/qa/qa.rb @@ -360,11 +360,11 @@ module QA autoload :Omnibus, 'qa/service/omnibus' autoload :Runner, 'qa/service/runner' - module CloudProvider - autoload :Base, 'qa/service/cloud_provider/base' - autoload :Gcloud, 'qa/service/cloud_provider/gcloud' - autoload :Minikube, 'qa/service/cloud_provider/minikube' - autoload :K3d, 'qa/service/cloud_provider/k3d' + module ClusterProvider + autoload :Base, 'qa/service/cluster_provider/base' + autoload :Gcloud, 'qa/service/cluster_provider/gcloud' + autoload :Minikube, 'qa/service/cluster_provider/minikube' + autoload :K3d, 'qa/service/cluster_provider/k3d' end end diff --git a/qa/qa/service/cloud_provider/base.rb b/qa/qa/service/cloud_provider/base.rb deleted file mode 100644 index 32b4d918f95..00000000000 --- a/qa/qa/service/cloud_provider/base.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module QA - module Service - module CloudProvider - class Base - include Service::Shellout - - attr_reader :rbac - - def initialize(rbac:) - @rbac = rbac - end - - def cluster_name - @cluster_name ||= "qa-cluster-#{Time.now.utc.strftime("%Y%m%d%H%M%S")}-#{SecureRandom.hex(4)}" - end - - def set_credentials(admin_user) - raise NotImplementedError - end - - def validate_dependencies - raise NotImplementedError - end - - def setup - raise NotImplementedError - end - - def teardown - raise NotImplementedError - end - - def filter_credentials(credentials) - credentials - end - end - end - end -end diff --git a/qa/qa/service/cloud_provider/gcloud.rb b/qa/qa/service/cloud_provider/gcloud.rb deleted file mode 100644 index 148c851636a..00000000000 --- a/qa/qa/service/cloud_provider/gcloud.rb +++ /dev/null @@ -1,91 +0,0 @@ -# frozen_string_literal: true - -module QA - module Service - module CloudProvider - class Gcloud < Base - def validate_dependencies - find_executable('gcloud') || raise("You must first install `gcloud` executable to run these tests.") - end - - def set_credentials(admin_user) - master_auth = JSON.parse(`gcloud container clusters describe #{cluster_name} --region #{region} --format 'json(masterAuth.username, masterAuth.password)'`) - - shell <<~CMD.tr("\n", ' ') - kubectl config set-credentials #{admin_user} - --username #{master_auth['masterAuth']['username']} - --password #{master_auth['masterAuth']['password']} - CMD - end - - def setup - login_if_not_already_logged_in - create_cluster - end - - def teardown - delete_cluster - end - - private - - def login_if_not_already_logged_in - if Runtime::Env.has_gcloud_credentials? - attempt_login_with_env_vars - else - account = `gcloud auth list --filter=status:ACTIVE --format="value(account)"` - if account.empty? - raise "Failed to login to gcloud. No credentials provided in environment and no credentials found locally." - else - puts "gcloud account found. Using: #{account} for creating K8s cluster." - end - end - end - - def attempt_login_with_env_vars - puts "No gcloud account. Attempting to login from env vars GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY." - gcloud_account_key = Tempfile.new('gcloud-account-key') - gcloud_account_key.write(Runtime::Env.gcloud_account_key) - gcloud_account_key.close - gcloud_account_email = Runtime::Env.gcloud_account_email - shell("gcloud auth activate-service-account #{gcloud_account_email} --key-file #{gcloud_account_key.path}") - ensure - gcloud_account_key && gcloud_account_key.unlink - end - - def auth_options - "--enable-legacy-authorization" unless rbac - end - - def create_cluster - shell <<~CMD.tr("\n", ' ') - gcloud container clusters - create #{cluster_name} - #{auth_options} - --enable-basic-auth - --region #{region} - --disk-size 10GB - --num-nodes #{Runtime::Env.gcloud_num_nodes} - && gcloud container clusters - get-credentials - --region #{region} - #{cluster_name} - CMD - end - - def delete_cluster - shell <<~CMD.tr("\n", ' ') - gcloud container clusters delete - --region #{region} - #{cluster_name} - --quiet --async - CMD - end - - def region - Runtime::Env.gcloud_region - end - end - end - end -end diff --git a/qa/qa/service/cloud_provider/k3d.rb b/qa/qa/service/cloud_provider/k3d.rb deleted file mode 100644 index 7fd2f73edb9..00000000000 --- a/qa/qa/service/cloud_provider/k3d.rb +++ /dev/null @@ -1,137 +0,0 @@ -# frozen_string_literal: true - -module QA - module Service - module CloudProvider - class K3d < Base - def validate_dependencies - find_executable('k3d') || raise("You must first install `k3d` executable to run these tests.") - end - - def set_credentials(admin_user) - end - - def setup - shell "k3d create --workers 1 --name #{cluster_name}" - - @old_kubeconfig = ENV['KUBECONFIG'] - ENV['KUBECONFIG'] = fetch_kubeconfig or raise "Could not fetch kubeconfig" - - wait_for_default_namespace - install_local_storage - end - - def teardown - ENV['KUBECONFIG'] = @old_kubeconfig - shell "k3d delete --name #{cluster_name}" - end - - # Fetch "real" certificate - # See https://github.com/rancher/k3s/issues/27 - def filter_credentials(credentials) - kubeconfig = YAML.load_file(ENV['KUBECONFIG']) - ca_certificate = kubeconfig.dig('clusters', 0, 'cluster', 'certificate-authority-data') - - credentials.merge('data' => credentials['data'].merge('ca.crt' => ca_certificate)) - end - - private - - def retry_until(max_attempts: 10, wait: 1) - max_attempts.times do - result = yield - return result if result - sleep wait - end - - raise "Retried #{max_attempts} times. Aborting" - end - - def fetch_kubeconfig - retry_until do - config = `k3d get-kubeconfig --name #{cluster_name}`.chomp - config if config =~ /kubeconfig.yaml/ - end - end - - def wait_for_default_namespace - retry_until do - result = `kubectl get namespace default` - result if result =~ /default/ - end - end - - def install_local_storage - shell('kubectl apply -f -', stdin_data: local_storage_config) - end - - # See https://github.com/rancher/k3d/issues/67 - def local_storage_config - <<~YAML - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: storage-provisioner - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: storage-provisioner - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:persistent-volume-provisioner - subjects: - - kind: ServiceAccount - name: storage-provisioner - namespace: kube-system - --- - apiVersion: v1 - kind: Pod - metadata: - name: storage-provisioner - namespace: kube-system - spec: - serviceAccountName: storage-provisioner - tolerations: - - effect: NoExecute - key: node.kubernetes.io/not-ready - operator: Exists - tolerationSeconds: 300 - - effect: NoExecute - key: node.kubernetes.io/unreachable - operator: Exists - tolerationSeconds: 300 - hostNetwork: true - containers: - - name: storage-provisioner - image: gcr.io/k8s-minikube/storage-provisioner:v1.8.1 - command: ["/storage-provisioner"] - imagePullPolicy: IfNotPresent - volumeMounts: - - mountPath: /tmp - name: tmp - volumes: - - name: tmp - hostPath: - path: /tmp - type: Directory - --- - kind: StorageClass - apiVersion: storage.k8s.io/v1 - metadata: - name: standard - namespace: kube-system - annotations: - storageclass.kubernetes.io/is-default-class: "true" - labels: - addonmanager.kubernetes.io/mode: EnsureExists - provisioner: k8s.io/minikube-hostpath - YAML - end - end - end - end -end diff --git a/qa/qa/service/cloud_provider/minikube.rb b/qa/qa/service/cloud_provider/minikube.rb deleted file mode 100644 index 08b2c7fbac9..00000000000 --- a/qa/qa/service/cloud_provider/minikube.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module QA - module Service - module CloudProvider - class Minikube < Base - def validate_dependencies - find_executable('minikube') || raise("You must first install `minikube` executable to run these tests.") - end - - def set_credentials(admin_user) - end - - def setup - shell 'minikube stop' - shell "minikube profile #{cluster_name}" - shell 'minikube start' - end - - def teardown - shell 'minikube delete' - end - end - end - end -end diff --git a/qa/qa/service/cluster_provider/base.rb b/qa/qa/service/cluster_provider/base.rb new file mode 100644 index 00000000000..a9678557aca --- /dev/null +++ b/qa/qa/service/cluster_provider/base.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module QA + module Service + module ClusterProvider + class Base + include Service::Shellout + + attr_reader :rbac + + def initialize(rbac:) + @rbac = rbac + end + + def cluster_name + @cluster_name ||= "qa-cluster-#{Time.now.utc.strftime("%Y%m%d%H%M%S")}-#{SecureRandom.hex(4)}" + end + + def set_credentials(admin_user) + raise NotImplementedError + end + + def validate_dependencies + raise NotImplementedError + end + + def setup + raise NotImplementedError + end + + def teardown + raise NotImplementedError + end + + def filter_credentials(credentials) + credentials + end + end + end + end +end diff --git a/qa/qa/service/cluster_provider/gcloud.rb b/qa/qa/service/cluster_provider/gcloud.rb new file mode 100644 index 00000000000..c30944966d0 --- /dev/null +++ b/qa/qa/service/cluster_provider/gcloud.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module QA + module Service + module ClusterProvider + class Gcloud < Base + def validate_dependencies + find_executable('gcloud') || raise("You must first install `gcloud` executable to run these tests.") + end + + def set_credentials(admin_user) + master_auth = JSON.parse(`gcloud container clusters describe #{cluster_name} --region #{region} --format 'json(masterAuth.username, masterAuth.password)'`) + + shell <<~CMD.tr("\n", ' ') + kubectl config set-credentials #{admin_user} + --username #{master_auth['masterAuth']['username']} + --password #{master_auth['masterAuth']['password']} + CMD + end + + def setup + login_if_not_already_logged_in + create_cluster + end + + def teardown + delete_cluster + end + + private + + def login_if_not_already_logged_in + if Runtime::Env.has_gcloud_credentials? + attempt_login_with_env_vars + else + account = `gcloud auth list --filter=status:ACTIVE --format="value(account)"` + if account.empty? + raise "Failed to login to gcloud. No credentials provided in environment and no credentials found locally." + else + puts "gcloud account found. Using: #{account} for creating K8s cluster." + end + end + end + + def attempt_login_with_env_vars + puts "No gcloud account. Attempting to login from env vars GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY." + gcloud_account_key = Tempfile.new('gcloud-account-key') + gcloud_account_key.write(Runtime::Env.gcloud_account_key) + gcloud_account_key.close + gcloud_account_email = Runtime::Env.gcloud_account_email + shell("gcloud auth activate-service-account #{gcloud_account_email} --key-file #{gcloud_account_key.path}") + ensure + gcloud_account_key && gcloud_account_key.unlink + end + + def auth_options + "--enable-legacy-authorization" unless rbac + end + + def create_cluster + shell <<~CMD.tr("\n", ' ') + gcloud container clusters + create #{cluster_name} + #{auth_options} + --enable-basic-auth + --region #{region} + --disk-size 10GB + --num-nodes #{Runtime::Env.gcloud_num_nodes} + && gcloud container clusters + get-credentials + --region #{region} + #{cluster_name} + CMD + end + + def delete_cluster + shell <<~CMD.tr("\n", ' ') + gcloud container clusters delete + --region #{region} + #{cluster_name} + --quiet --async + CMD + end + + def region + Runtime::Env.gcloud_region + end + end + end + end +end diff --git a/qa/qa/service/cluster_provider/k3d.rb b/qa/qa/service/cluster_provider/k3d.rb new file mode 100644 index 00000000000..81b1688d279 --- /dev/null +++ b/qa/qa/service/cluster_provider/k3d.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +module QA + module Service + module ClusterProvider + class K3d < Base + def validate_dependencies + find_executable('k3d') || raise("You must first install `k3d` executable to run these tests.") + end + + def set_credentials(admin_user) + end + + def setup + shell "k3d create --workers 1 --name #{cluster_name}" + + @old_kubeconfig = ENV['KUBECONFIG'] + ENV['KUBECONFIG'] = fetch_kubeconfig or raise "Could not fetch kubeconfig" + + wait_for_default_namespace + install_local_storage + end + + def teardown + ENV['KUBECONFIG'] = @old_kubeconfig + shell "k3d delete --name #{cluster_name}" + end + + # Fetch "real" certificate + # See https://github.com/rancher/k3s/issues/27 + def filter_credentials(credentials) + kubeconfig = YAML.load_file(ENV['KUBECONFIG']) + ca_certificate = kubeconfig.dig('clusters', 0, 'cluster', 'certificate-authority-data') + + credentials.merge('data' => credentials['data'].merge('ca.crt' => ca_certificate)) + end + + private + + def retry_until(max_attempts: 10, wait: 1) + max_attempts.times do + result = yield + return result if result + sleep wait + end + + raise "Retried #{max_attempts} times. Aborting" + end + + def fetch_kubeconfig + retry_until do + config = `k3d get-kubeconfig --name #{cluster_name}`.chomp + config if config =~ /kubeconfig.yaml/ + end + end + + def wait_for_default_namespace + retry_until do + result = `kubectl get namespace default` + result if result =~ /default/ + end + end + + def install_local_storage + shell('kubectl apply -f -', stdin_data: local_storage_config) + end + + # See https://github.com/rancher/k3d/issues/67 + def local_storage_config + <<~YAML + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: storage-provisioner + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: storage-provisioner + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:persistent-volume-provisioner + subjects: + - kind: ServiceAccount + name: storage-provisioner + namespace: kube-system + --- + apiVersion: v1 + kind: Pod + metadata: + name: storage-provisioner + namespace: kube-system + spec: + serviceAccountName: storage-provisioner + tolerations: + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 300 + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 300 + hostNetwork: true + containers: + - name: storage-provisioner + image: gcr.io/k8s-minikube/storage-provisioner:v1.8.1 + command: ["/storage-provisioner"] + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /tmp + name: tmp + volumes: + - name: tmp + hostPath: + path: /tmp + type: Directory + --- + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: standard + namespace: kube-system + annotations: + storageclass.kubernetes.io/is-default-class: "true" + labels: + addonmanager.kubernetes.io/mode: EnsureExists + provisioner: k8s.io/minikube-hostpath + YAML + end + end + end + end +end diff --git a/qa/qa/service/cluster_provider/minikube.rb b/qa/qa/service/cluster_provider/minikube.rb new file mode 100644 index 00000000000..fc916245357 --- /dev/null +++ b/qa/qa/service/cluster_provider/minikube.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module QA + module Service + module ClusterProvider + class Minikube < Base + def validate_dependencies + find_executable('minikube') || raise("You must first install `minikube` executable to run these tests.") + end + + def set_credentials(admin_user) + end + + def setup + shell 'minikube stop' + shell "minikube profile #{cluster_name}" + shell 'minikube start' + end + + def teardown + shell 'minikube delete' + end + end + end + end +end diff --git a/qa/qa/service/kubernetes_cluster.rb b/qa/qa/service/kubernetes_cluster.rb index f83def801ce..31138a13959 100644 --- a/qa/qa/service/kubernetes_cluster.rb +++ b/qa/qa/service/kubernetes_cluster.rb @@ -13,9 +13,9 @@ module QA def initialize(rbac: true) @rbac = rbac - @provider = QA::Service::CloudProvider::Gcloud.new(rbac: rbac) - # @provider = QA::Service::CloudProvider::Minikube.new(rbac: rbac) - # @provider = QA::Service::CloudProvider::K3d.new(rbac: rbac) + @provider = QA::Service::ClusterProvider::Gcloud.new(rbac: rbac) + # @provider = QA::Service::ClusterProvider::Minikube.new(rbac: rbac) + # @provider = QA::Service::ClusterProvider::K3d.new(rbac: rbac) end def create! -- cgit v1.2.1