summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorRémy Coutable <remy@rymai.me>2018-09-25 16:32:53 +0200
committerRémy Coutable <remy@rymai.me>2018-09-25 16:41:27 +0200
commit3299680cdd084746f85bcb662523ed6bfe2030f2 (patch)
tree97fe04bd6ed9f792dcebc2a28223b1c9f57c4aca /scripts
parentcb0f024cd4c4aa51c7824e4e1a6c4d0f307afad1 (diff)
downloadgitlab-ce-3299680cdd084746f85bcb662523ed6bfe2030f2.tar.gz
[CE] Port review apps file to CEce-improve-review-apps
Signed-off-by: Rémy Coutable <remy@rymai.me>
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/review_apps/automated_cleanup.rb109
-rwxr-xr-xscripts/review_apps/review-apps.sh184
2 files changed, 293 insertions, 0 deletions
diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb
new file mode 100755
index 00000000000..ea53f89c844
--- /dev/null
+++ b/scripts/review_apps/automated_cleanup.rb
@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+require_relative File.expand_path('../../lib/quality/helm_client.rb', __dir__)
+require_relative File.expand_path('../../lib/quality/kubernetes_client.rb', __dir__)
+
+class AutomatedCleanup
+ attr_reader :project_path, :gitlab_token, :cleaned_up_releases
+
+ def initialize(project_path: ENV['CI_PROJECT_PATH'], gitlab_token: ENV['GITLAB_BOT_REVIEW_APPS_CLEANUP_TOKEN'])
+ @project_path = project_path
+ @gitlab_token = gitlab_token
+ @cleaned_up_releases = []
+ end
+
+ def gitlab
+ @gitlab ||= begin
+ Gitlab.configure do |config|
+ config.endpoint = 'https://gitlab.com/api/v4'
+ # gitlab-bot's token "GitLab review apps cleanup"
+ config.private_token = gitlab_token
+ end
+
+ Gitlab
+ end
+ end
+
+ def helm
+ @helm ||= Quality::HelmClient.new
+ end
+
+ def kubernetes
+ @kubernetes ||= Quality::KubernetesClient.new
+ end
+
+ def perform_gitlab_environment_cleanup!(days_for_stop:, days_for_delete:)
+ puts "Checking for review apps not updated in the last #{days_for_stop} days..."
+
+ checked_environments = []
+ delete_threshold = threshold_time(days: days_for_delete)
+ stop_threshold = threshold_time(days: days_for_stop)
+ gitlab.deployments(project_path, per_page: 50).auto_paginate do |deployment|
+ next unless deployment.environment.name.start_with?('review/')
+ next if checked_environments.include?(deployment.environment.slug)
+
+ puts
+
+ checked_environments << deployment.environment.slug
+ deployed_at = Time.parse(deployment.created_at)
+
+ if deployed_at < delete_threshold
+ print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'deleting')
+ gitlab.delete_environment(project_path, deployment.environment.id)
+ cleaned_up_releases << deployment.environment.slug
+ elsif deployed_at < stop_threshold
+ print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'stopping')
+ gitlab.stop_environment(project_path, deployment.environment.id)
+ cleaned_up_releases << deployment.environment.slug
+ else
+ print_release_state(subject: 'Review app', release_name: deployment.environment.slug, release_date: deployment.created_at, action: 'leaving')
+ end
+ end
+ end
+
+ def perform_helm_releases_cleanup!(days:)
+ puts "Checking for Helm releases not updated in the last #{days} days..."
+
+ threshold_day = threshold_time(days: days)
+ helm.releases(args: ['--deployed', '--failed', '--date', '--reverse', '--max 25']).each do |release|
+ next if cleaned_up_releases.include?(release.name)
+
+ if release.last_update < threshold_day
+ print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'cleaning')
+ helm.delete(release_name: release.name)
+ kubernetes.cleanup(release_name: release.name)
+ else
+ print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'leaving')
+ end
+ end
+ end
+
+ def threshold_time(days:)
+ Time.now - days * 24 * 3600
+ end
+
+ def print_release_state(subject:, release_name:, release_date:, action:)
+ puts "\n#{subject} '#{release_name}' was last deployed on #{release_date}: #{action} it."
+ end
+end
+
+def timed(task)
+ start = Time.now
+ yield(self)
+ puts "#{task} finished in #{Time.now - start} seconds.\n"
+end
+
+automated_cleanup = AutomatedCleanup.new
+
+timed('Review apps cleanup') do
+ automated_cleanup.perform_gitlab_environment_cleanup!(days_for_stop: 5, days_for_delete: 6)
+end
+
+puts
+
+timed('Helm releases cleanup') do
+ automated_cleanup.perform_helm_releases_cleanup!(days: 7)
+end
+
+exit(0)
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
new file mode 100755
index 00000000000..78293464265
--- /dev/null
+++ b/scripts/review_apps/review-apps.sh
@@ -0,0 +1,184 @@
+[[ "$TRACE" ]] && set -x
+export TILLER_NAMESPACE="$KUBE_NAMESPACE"
+
+function check_kube_domain() {
+ if [ -z ${REVIEW_APPS_DOMAIN+x} ]; then
+ echo "In order to deploy or use Review Apps, REVIEW_APPS_DOMAIN variable must be set"
+ echo "You can do it in Auto DevOps project settings or defining a variable at group or project level"
+ echo "You can also manually add it in .gitlab-ci.yml"
+ false
+ else
+ true
+ fi
+}
+
+function download_gitlab_chart() {
+ curl -o gitlab.tar.bz2 https://gitlab.com/charts/gitlab/-/archive/$GITLAB_HELM_CHART_REF/gitlab-$GITLAB_HELM_CHART_REF.tar.bz2
+ tar -xjf gitlab.tar.bz2
+ cd gitlab-$GITLAB_HELM_CHART_REF
+
+ helm init --client-only
+ helm repo add gitlab https://charts.gitlab.io
+ helm dependency update
+ helm dependency build
+}
+
+function ensure_namespace() {
+ kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"
+}
+
+function install_tiller() {
+ echo "Checking Tiller..."
+ helm init --upgrade
+ kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy"
+ if ! helm version --debug; then
+ echo "Failed to init Tiller."
+ return 1
+ fi
+ echo ""
+}
+
+function create_secret() {
+ echo "Create secret..."
+
+ kubectl create secret generic -n "$KUBE_NAMESPACE" \
+ $CI_ENVIRONMENT_SLUG-gitlab-initial-root-password \
+ --from-literal=password=$REVIEW_APPS_ROOT_PASSWORD \
+ --dry-run -o json | kubectl apply -f -
+}
+
+function previousDeployFailed() {
+ set +e
+ echo "Checking for previous deployment of $CI_ENVIRONMENT_SLUG"
+ deployment_status=$(helm status $CI_ENVIRONMENT_SLUG >/dev/null 2>&1)
+ status=$?
+ # if `status` is `0`, deployment exists, has a status
+ if [ $status -eq 0 ]; then
+ echo "Previous deployment found, checking status"
+ deployment_status=$(helm status $CI_ENVIRONMENT_SLUG | grep ^STATUS | cut -d' ' -f2)
+ echo "Previous deployment state: $deployment_status"
+ if [[ "$deployment_status" == "FAILED" || "$deployment_status" == "PENDING_UPGRADE" || "$deployment_status" == "PENDING_INSTALL" ]]; then
+ status=0;
+ else
+ status=1;
+ fi
+ else
+ echo "Previous deployment NOT found."
+ fi
+ set -e
+ return $status
+}
+
+function deploy() {
+ track="${1-stable}"
+ name="$CI_ENVIRONMENT_SLUG"
+
+ if [[ "$track" != "stable" ]]; then
+ name="$name-$track"
+ fi
+
+ replicas="1"
+ service_enabled="false"
+ postgres_enabled="$POSTGRES_ENABLED"
+ gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ce"
+ gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ce"
+ gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ce"
+ gitlab_gitaly_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly"
+ gitlab_shell_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell"
+ gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ce"
+
+ if [[ "$CI_PROJECT_NAME" == "gitlab-ee" ]]; then
+ gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ee"
+ gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ee"
+ gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ee"
+ gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ee"
+ fi
+
+ # canary uses stable db
+ [[ "$track" == "canary" ]] && postgres_enabled="false"
+
+ env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' )
+ env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' )
+
+ if [[ "$track" == "stable" ]]; then
+ # for stable track get number of replicas from `PRODUCTION_REPLICAS`
+ eval new_replicas=\$${env_slug}_REPLICAS
+ service_enabled="true"
+ else
+ # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS`
+ eval new_replicas=\$${env_track}_${env_slug}_REPLICAS
+ fi
+ if [[ -n "$new_replicas" ]]; then
+ replicas="$new_replicas"
+ fi
+
+ # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
+ if [ "$CI_ENVIRONMENT_SLUG" != "production" ] && previousDeployFailed ; then
+ echo "Deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG"
+ delete
+ cleanup
+ fi
+ helm repo add gitlab https://charts.gitlab.io/
+ helm dep update .
+
+HELM_CMD=$(cat << EOF
+ helm upgrade --install \
+ --wait \
+ --timeout 600 \
+ --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
+ --set global.hosts.hostSuffix="$HOST_SUFFIX" \
+ --set global.hosts.domain="$REVIEW_APPS_DOMAIN" \
+ --set certmanager.install=false \
+ --set global.ingress.configureCertmanager=false \
+ --set global.ingress.tls.secretName=tls-cert \
+ --set global.ingress.annotations."external-dns\.alpha\.kubernetes\.io/ttl"="10"
+ --set gitlab.unicorn.resources.requests.cpu=200m \
+ --set gitlab.sidekiq.resources.requests.cpu=100m \
+ --set gitlab.gitlab-shell.resources.requests.cpu=100m \
+ --set redis.resources.requests.cpu=100m \
+ --set minio.resources.requests.cpu=100m \
+ --set gitlab.migrations.image.repository="$gitlab_migrations_image_repository" \
+ --set gitlab.migrations.image.tag="$CI_COMMIT_REF_NAME" \
+ --set gitlab.sidekiq.image.repository="$gitlab_sidekiq_image_repository" \
+ --set gitlab.sidekiq.image.tag="$CI_COMMIT_REF_NAME" \
+ --set gitlab.unicorn.image.repository="$gitlab_unicorn_image_repository" \
+ --set gitlab.unicorn.image.tag="$CI_COMMIT_REF_NAME" \
+ --set gitlab.gitaly.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly" \
+ --set gitlab.gitaly.image.tag="v$GITALY_VERSION" \
+ --set gitlab.gitlab-shell.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell" \
+ --set gitlab.gitlab-shell.image.tag="v$GITLAB_SHELL_VERSION" \
+ --set gitlab.unicorn.workhorse.image="$gitlab_workhorse_image_repository" \
+ --set gitlab.unicorn.workhorse.tag="$CI_COMMIT_REF_NAME" \
+ --namespace="$KUBE_NAMESPACE" \
+ --version="$CI_PIPELINE_ID-$CI_JOB_ID" \
+ "$name" \
+ .
+EOF
+)
+
+ echo "Deploying with:"
+ echo $HELM_CMD
+
+ eval $HELM_CMD
+}
+
+function delete() {
+ track="${1-stable}"
+ name="$CI_ENVIRONMENT_SLUG"
+
+ if [[ "$track" != "stable" ]]; then
+ name="$name-$track"
+ fi
+
+ echo "Deleting release '$name'..."
+ helm delete --purge "$name" || true
+}
+
+function cleanup() {
+ echo "Cleaning up $CI_ENVIRONMENT_SLUG..."
+ kubectl -n "$KUBE_NAMESPACE" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1 \
+ | grep "$CI_ENVIRONMENT_SLUG" \
+ | awk '{print $1}' \
+ | xargs kubectl -n "$KUBE_NAMESPACE" delete \
+ || true
+}