diff options
Diffstat (limited to 'qa/qa/service')
| -rw-r--r-- | qa/qa/service/cluster_provider/k3s_cilium.rb | 93 | ||||
| -rw-r--r-- | qa/qa/service/docker_run/gitlab_runner.rb | 16 | ||||
| -rw-r--r-- | qa/qa/service/docker_run/k3s.rb | 15 | ||||
| -rw-r--r-- | qa/qa/service/kubernetes_cluster.rb | 24 |
4 files changed, 144 insertions, 4 deletions
diff --git a/qa/qa/service/cluster_provider/k3s_cilium.rb b/qa/qa/service/cluster_provider/k3s_cilium.rb new file mode 100644 index 00000000000..5b529caa20b --- /dev/null +++ b/qa/qa/service/cluster_provider/k3s_cilium.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module QA + module Service + module ClusterProvider + class K3sCilium < K3s + def setup + @k3s = Service::DockerRun::K3s.new.tap do |k3s| + k3s.remove! + k3s.cni_enabled = true + k3s.register! + + shell "kubectl config set-cluster k3s --server https://#{k3s.host_name}:6443 --insecure-skip-tls-verify" + shell 'kubectl config set-credentials default --username=node --password=some-secret' + shell 'kubectl config set-context k3s --cluster=k3s --user=default' + shell 'kubectl config use-context k3s' + + wait_for_server(k3s.host_name) do + shell 'kubectl version' + # install local storage + shell 'kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/deploy/local-path-storage.yaml' + + # patch local storage + shell %(kubectl patch storageclass local-path -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}') + shell 'kubectl create -f https://raw.githubusercontent.com/cilium/cilium/v1.8/install/kubernetes/quick-install.yaml' + + wait_for_namespaces do + wait_for_cilium + wait_for_coredns do + shell 'kubectl create -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-0.31.0/deploy/static/provider/cloud/deploy.yaml' + wait_for_ingress + end + end + end + end + end + + private + + def wait_for_cilium + QA::Runtime::Logger.info 'Waiting for Cilium pod to be initialized' + + 60.times do + if service_available?('kubectl get pods --all-namespaces -l k8s-app=cilium --no-headers=true | grep -o "cilium-.*1/1"') + return yield if block_given? + + return true + end + + sleep 1 + QA::Runtime::Logger.info '.' + end + + raise 'Cilium pod has not initialized correctly' + end + + def wait_for_coredns + QA::Runtime::Logger.info 'Waiting for CoreDNS pod to be initialized' + + 60.times do + if service_available?('kubectl get pods --all-namespaces --no-headers=true | grep -o "coredns.*1/1"') + return yield if block_given? + + return true + end + + sleep 1 + QA::Runtime::Logger.info '.' + end + + raise 'CoreDNS pod has not been initialized correctly' + end + + def wait_for_ingress + QA::Runtime::Logger.info 'Waiting for Ingress controller pod to be initialized' + + 60.times do + if service_available?('kubectl get pods --all-namespaces -l app.kubernetes.io/component=controller | grep -o "ingress-nginx-controller.*1/1"') + return yield if block_given? + + return true + end + + sleep 1 + QA::Runtime::Logger.info '.' + end + + raise 'Ingress pod has not been initialized correctly' + end + end + end + end +end diff --git a/qa/qa/service/docker_run/gitlab_runner.rb b/qa/qa/service/docker_run/gitlab_runner.rb index a5b129eb1f9..63fbf758231 100644 --- a/qa/qa/service/docker_run/gitlab_runner.rb +++ b/qa/qa/service/docker_run/gitlab_runner.rb @@ -38,11 +38,10 @@ module QA def register! shell <<~CMD.tr("\n", ' ') - docker run -d --rm --entrypoint=/bin/sh - --network #{runner_network} --name #{@name} + docker run -d --rm --network #{runner_network} --name #{@name} #{'-v /var/run/docker.sock:/var/run/docker.sock' if @executor == :docker} --privileged - #{@image} -c "#{register_command}" + #{@image} #{add_gitlab_tls_cert if @address.include? "https"} && docker exec --detach #{@name} sh -c "#{register_command}" CMD # Prove airgappedness @@ -82,6 +81,7 @@ module QA args << '--docker-tlsverify=false' args << '--docker-privileged=true' args << "--docker-network-mode=#{network}" + args << "--docker-volumes=/certs/client" end <<~CMD.strip @@ -102,6 +102,16 @@ module QA wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 && (echo "Airgapped network faulty. Connectivity wget check failed." && exit 1) || (echo "Airgapped network confirmed. Connectivity wget check passed." && exit 0) CMD end + + def add_gitlab_tls_cert + gitlab_tls_certificate = Tempfile.new('gitlab-cert') + gitlab_tls_certificate.write(Runtime::Env.gitlab_tls_certificate) + gitlab_tls_certificate.close + + <<~CMD + && docker cp #{gitlab_tls_certificate.path} #{@name}:/etc/gitlab-runner/certs/gitlab.test.crt + CMD + end end end end diff --git a/qa/qa/service/docker_run/k3s.rb b/qa/qa/service/docker_run/k3s.rb index 07211b220f1..a09b62cb613 100644 --- a/qa/qa/service/docker_run/k3s.rb +++ b/qa/qa/service/docker_run/k3s.rb @@ -4,15 +4,20 @@ module QA module Service module DockerRun class K3s < Base + attr_accessor :cni_enabled + def initialize - @image = 'registry.gitlab.com/gitlab-org/cluster-integration/test-utils/k3s-gitlab-ci/releases/v0.6.1' + @image = 'registry.gitlab.com/gitlab-org/cluster-integration/test-utils/k3s-gitlab-ci/releases/v0.9.1' @name = 'k3s' + @cni_enabled = false super end def register! pull start_k3s + # Mount the berkeley packet filter if container network interface is enabled + mount_bpf if @cni_enabled end def host_name @@ -36,12 +41,20 @@ module QA #{@image} server --cluster-secret some-secret --no-deploy traefik + #{@cni_enabled ? '--no-flannel' : ''} CMD command.gsub!("--network #{network} --hostname #{host_name}", '') unless QA::Runtime::Env.running_in_ci? shell command end + + private + + def mount_bpf + shell "docker exec --privileged k3s mount bpffs -t bpf /sys/fs/bpf" + shell "docker exec --privileged k3s mount --make-shared bpffs -t bpf /sys/fs/bpf" + end end end end diff --git a/qa/qa/service/kubernetes_cluster.rb b/qa/qa/service/kubernetes_cluster.rb index ddf97046fb0..adef1b46af2 100644 --- a/qa/qa/service/kubernetes_cluster.rb +++ b/qa/qa/service/kubernetes_cluster.rb @@ -51,6 +51,30 @@ module QA shell('kubectl apply -f -', stdin_data: manifest) end + def add_sample_policy(project, policy_name: 'sample-policy') + namespace = "#{project.name}-#{project.id}-production" + network_policy = <<~YAML + apiVersion: "cilium.io/v2" + kind: CiliumNetworkPolicy + metadata: + name: #{policy_name} + namespace: #{namespace} + spec: + endpointSelector: + matchLabels: + role: backend + ingress: + - fromEndpoints: + - matchLabels: + role: frontend + YAML + shell('kubectl apply -f -', stdin_data: network_policy) + end + + def fetch_external_ip_for_ingress + `kubectl get svc --all-namespaces --no-headers=true -l app.kubernetes.io/name=ingress-nginx -o custom-columns=:'status.loadBalancer.ingress[0].ip' | grep -v 'none'` + end + private def fetch_api_url |