From 3299680cdd084746f85bcb662523ed6bfe2030f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 25 Sep 2018 16:32:53 +0200 Subject: [CE] Port review apps file to CE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- scripts/review_apps/automated_cleanup.rb | 109 ++++++++++++++++++ scripts/review_apps/review-apps.sh | 184 +++++++++++++++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100755 scripts/review_apps/automated_cleanup.rb create mode 100755 scripts/review_apps/review-apps.sh (limited to 'scripts/review_apps') diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb new file mode 100755 index 00000000000..ea53f89c844 --- /dev/null +++ b/scripts/review_apps/automated_cleanup.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +require 'gitlab' +require_relative File.expand_path('../../lib/quality/helm_client.rb', __dir__) +require_relative File.expand_path('../../lib/quality/kubernetes_client.rb', __dir__) + +class AutomatedCleanup + attr_reader :project_path, :gitlab_token, :cleaned_up_releases + + def initialize(project_path: ENV['CI_PROJECT_PATH'], gitlab_token: ENV['GITLAB_BOT_REVIEW_APPS_CLEANUP_TOKEN']) + @project_path = project_path + @gitlab_token = gitlab_token + @cleaned_up_releases = [] + end + + def gitlab + @gitlab ||= begin + Gitlab.configure do |config| + config.endpoint = 'https://gitlab.com/api/v4' + # gitlab-bot's token "GitLab review apps cleanup" + config.private_token = gitlab_token + end + + Gitlab + end + end + + def helm + @helm ||= Quality::HelmClient.new + end + + def kubernetes + @kubernetes ||= Quality::KubernetesClient.new + end + + def perform_gitlab_environment_cleanup!(days_for_stop:, days_for_delete:) + puts "Checking for review apps not updated in the last #{days_for_stop} days..." + + checked_environments = [] + delete_threshold = threshold_time(days: days_for_delete) + stop_threshold = threshold_time(days: days_for_stop) + gitlab.deployments(project_path, per_page: 50).auto_paginate do |deployment| + next unless deployment.environment.name.start_with?('review/') + next if checked_environments.include?(deployment.environment.slug) + + puts + + checked_environments << deployment.environment.slug + deployed_at = Time.parse(deployment.created_at) + + if deployed_at < delete_threshold + print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'deleting') + gitlab.delete_environment(project_path, deployment.environment.id) + cleaned_up_releases << deployment.environment.slug + elsif deployed_at < stop_threshold + print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'stopping') + gitlab.stop_environment(project_path, deployment.environment.id) + cleaned_up_releases << deployment.environment.slug + else + print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'leaving') + end + end + end + + def perform_helm_releases_cleanup!(days:) + puts "Checking for Helm releases not updated in the last #{days} days..." + + threshold_day = threshold_time(days: days) + helm.releases(args: ['--deployed', '--failed', '--date', '--reverse', '--max 25']).each do |release| + next if cleaned_up_releases.include?(release.name) + + if release.last_update < threshold_day + print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'cleaning') + helm.delete(release_name: release.name) + kubernetes.cleanup(release_name: release.name) + else + print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'leaving') + end + end + end + + def threshold_time(days:) + Time.now - days * 24 * 3600 + end + + def print_release_state(subject:, release_name:, release_date:, action:) + puts "\n#{subject} '#{release_name}' was last deployed on #{release_date}: #{action} it." + end +end + +def timed(task) + start = Time.now + yield(self) + puts "#{task} finished in #{Time.now - start} seconds.\n" +end + +automated_cleanup = AutomatedCleanup.new + +timed('Review apps cleanup') do + automated_cleanup.perform_gitlab_environment_cleanup!(days_for_stop: 5, days_for_delete: 6) +end + +puts + +timed('Helm releases cleanup') do + automated_cleanup.perform_helm_releases_cleanup!(days: 7) +end + +exit(0) diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh new file mode 100755 index 00000000000..78293464265 --- /dev/null +++ b/scripts/review_apps/review-apps.sh @@ -0,0 +1,184 @@ +[[ "$TRACE" ]] && set -x +export TILLER_NAMESPACE="$KUBE_NAMESPACE" + +function check_kube_domain() { + if [ -z ${REVIEW_APPS_DOMAIN+x} ]; then + echo "In order to deploy or use Review Apps, REVIEW_APPS_DOMAIN variable must be set" + echo "You can do it in Auto DevOps project settings or defining a variable at group or project level" + echo "You can also manually add it in .gitlab-ci.yml" + false + else + true + fi +} + +function download_gitlab_chart() { + curl -o gitlab.tar.bz2 https://gitlab.com/charts/gitlab/-/archive/$GITLAB_HELM_CHART_REF/gitlab-$GITLAB_HELM_CHART_REF.tar.bz2 + tar -xjf gitlab.tar.bz2 + cd gitlab-$GITLAB_HELM_CHART_REF + + helm init --client-only + helm repo add gitlab https://charts.gitlab.io + helm dependency update + helm dependency build +} + +function ensure_namespace() { + kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" +} + +function install_tiller() { + echo "Checking Tiller..." + helm init --upgrade + kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy" + if ! helm version --debug; then + echo "Failed to init Tiller." + return 1 + fi + echo "" +} + +function create_secret() { + echo "Create secret..." + + kubectl create secret generic -n "$KUBE_NAMESPACE" \ + $CI_ENVIRONMENT_SLUG-gitlab-initial-root-password \ + --from-literal=password=$REVIEW_APPS_ROOT_PASSWORD \ + --dry-run -o json | kubectl apply -f - +} + +function previousDeployFailed() { + set +e + echo "Checking for previous deployment of $CI_ENVIRONMENT_SLUG" + deployment_status=$(helm status $CI_ENVIRONMENT_SLUG >/dev/null 2>&1) + status=$? + # if `status` is `0`, deployment exists, has a status + if [ $status -eq 0 ]; then + echo "Previous deployment found, checking status" + deployment_status=$(helm status $CI_ENVIRONMENT_SLUG | grep ^STATUS | cut -d' ' -f2) + echo "Previous deployment state: $deployment_status" + if [[ "$deployment_status" == "FAILED" || "$deployment_status" == "PENDING_UPGRADE" || "$deployment_status" == "PENDING_INSTALL" ]]; then + status=0; + else + status=1; + fi + else + echo "Previous deployment NOT found." + fi + set -e + return $status +} + +function deploy() { + track="${1-stable}" + name="$CI_ENVIRONMENT_SLUG" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + replicas="1" + service_enabled="false" + postgres_enabled="$POSTGRES_ENABLED" + gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ce" + gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ce" + gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ce" + gitlab_gitaly_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly" + gitlab_shell_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell" + gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ce" + + if [[ "$CI_PROJECT_NAME" == "gitlab-ee" ]]; then + gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ee" + gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ee" + gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ee" + gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ee" + fi + + # canary uses stable db + [[ "$track" == "canary" ]] && postgres_enabled="false" + + env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' ) + env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' ) + + if [[ "$track" == "stable" ]]; then + # for stable track get number of replicas from `PRODUCTION_REPLICAS` + eval new_replicas=\$${env_slug}_REPLICAS + service_enabled="true" + else + # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` + eval new_replicas=\$${env_track}_${env_slug}_REPLICAS + fi + if [[ -n "$new_replicas" ]]; then + replicas="$new_replicas" + fi + + # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade` + if [ "$CI_ENVIRONMENT_SLUG" != "production" ] && previousDeployFailed ; then + echo "Deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG" + delete + cleanup + fi + helm repo add gitlab https://charts.gitlab.io/ + helm dep update . + +HELM_CMD=$(cat << EOF + helm upgrade --install \ + --wait \ + --timeout 600 \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set global.hosts.hostSuffix="$HOST_SUFFIX" \ + --set global.hosts.domain="$REVIEW_APPS_DOMAIN" \ + --set certmanager.install=false \ + --set global.ingress.configureCertmanager=false \ + --set global.ingress.tls.secretName=tls-cert \ + --set global.ingress.annotations."external-dns\.alpha\.kubernetes\.io/ttl"="10" + --set gitlab.unicorn.resources.requests.cpu=200m \ + --set gitlab.sidekiq.resources.requests.cpu=100m \ + --set gitlab.gitlab-shell.resources.requests.cpu=100m \ + --set redis.resources.requests.cpu=100m \ + --set minio.resources.requests.cpu=100m \ + --set gitlab.migrations.image.repository="$gitlab_migrations_image_repository" \ + --set gitlab.migrations.image.tag="$CI_COMMIT_REF_NAME" \ + --set gitlab.sidekiq.image.repository="$gitlab_sidekiq_image_repository" \ + --set gitlab.sidekiq.image.tag="$CI_COMMIT_REF_NAME" \ + --set gitlab.unicorn.image.repository="$gitlab_unicorn_image_repository" \ + --set gitlab.unicorn.image.tag="$CI_COMMIT_REF_NAME" \ + --set gitlab.gitaly.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly" \ + --set gitlab.gitaly.image.tag="v$GITALY_VERSION" \ + --set gitlab.gitlab-shell.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell" \ + --set gitlab.gitlab-shell.image.tag="v$GITLAB_SHELL_VERSION" \ + --set gitlab.unicorn.workhorse.image="$gitlab_workhorse_image_repository" \ + --set gitlab.unicorn.workhorse.tag="$CI_COMMIT_REF_NAME" \ + --namespace="$KUBE_NAMESPACE" \ + --version="$CI_PIPELINE_ID-$CI_JOB_ID" \ + "$name" \ + . +EOF +) + + echo "Deploying with:" + echo $HELM_CMD + + eval $HELM_CMD +} + +function delete() { + track="${1-stable}" + name="$CI_ENVIRONMENT_SLUG" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + echo "Deleting release '$name'..." + helm delete --purge "$name" || true +} + +function cleanup() { + echo "Cleaning up $CI_ENVIRONMENT_SLUG..." + kubectl -n "$KUBE_NAMESPACE" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1 \ + | grep "$CI_ENVIRONMENT_SLUG" \ + | awk '{print $1}' \ + | xargs kubectl -n "$KUBE_NAMESPACE" delete \ + || true +} -- cgit v1.2.1