summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-08-20 18:42:06 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2020-08-20 18:42:06 +0000
commit6e4e1050d9dba2b7b2523fdd1768823ab85feef4 (patch)
tree78be5963ec075d80116a932011d695dd33910b4e /scripts
parent1ce776de4ae122aba3f349c02c17cebeaa8ecf07 (diff)
downloadgitlab-ce-6e4e1050d9dba2b7b2523fdd1768823ab85feef4.tar.gz
Add latest changes from gitlab-org/gitlab@13-3-stable-ee
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/prepare_postgres_fdw.sh15
-rwxr-xr-xscripts/regenerate-schema38
-rwxr-xr-xscripts/review_apps/review-apps.sh4
-rwxr-xr-xscripts/schema_changed.sh10
-rwxr-xr-xscripts/slack17
-rw-r--r--scripts/sync-stable-branch.sh3
-rwxr-xr-xscripts/trigger-build150
-rwxr-xr-xscripts/trigger-build-docs136
8 files changed, 198 insertions, 175 deletions
diff --git a/scripts/prepare_postgres_fdw.sh b/scripts/prepare_postgres_fdw.sh
deleted file mode 100755
index 246f3acc569..00000000000
--- a/scripts/prepare_postgres_fdw.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-psql -h postgres -U postgres gitlabhq_geo_test <<EOF
-CREATE EXTENSION postgres_fdw;
-CREATE SERVER gitlab_secondary FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host 'localhost', dbname 'gitlabhq_test');
-CREATE USER MAPPING FOR current_user SERVER gitlab_secondary OPTIONS (user 'postgres', password '');
-CREATE SCHEMA gitlab_secondary;
-IMPORT FOREIGN SCHEMA public FROM SERVER gitlab_secondary INTO gitlab_secondary;
-GRANT USAGE ON FOREIGN SERVER gitlab_secondary TO current_user;
-EOF
-
-# Ensure the FDW setting is enabled
-sed -i '/fdw:/d' config/database_geo.yml
-sed -i '/gitlabhq_geo_test/a\
-\ \ fdw: true' config/database_geo.yml
diff --git a/scripts/regenerate-schema b/scripts/regenerate-schema
index cedd612f766..e2d46d14218 100755
--- a/scripts/regenerate-schema
+++ b/scripts/regenerate-schema
@@ -3,6 +3,7 @@
# frozen_string_literal: true
require 'open3'
+require 'fileutils'
require 'uri'
class SchemaRegenerator
@@ -19,11 +20,19 @@ class SchemaRegenerator
# these to disable/enable migrations.
MIGRATION_DIRS = %w[db/migrate db/post_migrate].freeze
+ ##
+ # Directory where we store schema versions
+ #
+ # The remove_schema_migration_files removes files added in this
+ # directory when it runs.
+ SCHEMA_MIGRATIONS_DIR = 'db/schema_migrations/'
+
def execute
Dir.chdir(File.expand_path('..', __dir__)) do
checkout_ref
checkout_clean_schema
hide_migrations
+ remove_schema_migration_files
reset_db
unhide_migrations
migrate
@@ -111,6 +120,35 @@ class SchemaRegenerator
end
##
+ # Remove files added to db/schema_migrations
+ #
+ # In order to properly reset the database and re-run migrations
+ # the schema migrations for new migrations must be removed.
+ def remove_schema_migration_files
+ (untracked_schema_migrations + commited_schema_migrations).each do |schema_migration|
+ FileUtils.rm(schema_migration)
+ end
+ end
+
+ ##
+ # List of untracked schema migrations
+ #
+ # Get a list of schema migrations that are not tracked so we can remove them
+ def untracked_schema_migrations
+ git_command = "git ls-files --others --exclude-standard -- #{SCHEMA_MIGRATIONS_DIR}"
+ run(git_command).chomp.split("\n")
+ end
+
+ ##
+ # List of untracked schema migrations
+ #
+ # Get a list of schema migrations that have been committed since the last
+ def commited_schema_migrations
+ git_command = "git diff --name-only --diff-filter=A #{merge_base} -- #{SCHEMA_MIGRATIONS_DIR}"
+ run(git_command).chomp.split("\n")
+ end
+
+ ##
# Run rake task to reset the database.
def reset_db
run %q[bin/rails db:reset RAILS_ENV=test]
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index 74291f6aef4..59189c94cde 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -147,9 +147,9 @@ function disable_sign_ups() {
run_task "${ruby_cmd}"
# Disable sign-ups
- curl --silent --show-error --request PUT --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings?signup_enabled=false"
+ retry 'curl --silent --show-error --request PUT --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings?signup_enabled=false"'
- local signup_enabled=$(curl --silent --show-error --request GET --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings" | jq ".signup_enabled")
+ local signup_enabled=$(retry 'curl --silent --show-error --request GET --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings" | jq ".signup_enabled"')
if [[ "${signup_enabled}" == "false" ]]; then
echoinfo "Sign-ups have been disabled successfully."
else
diff --git a/scripts/schema_changed.sh b/scripts/schema_changed.sh
index 427e0128df7..f564f717e95 100755
--- a/scripts/schema_changed.sh
+++ b/scripts/schema_changed.sh
@@ -10,6 +10,16 @@ schema_changed() {
else
printf "Schema changes are correctly applied to db/structure.sql\n"
fi
+
+ if [ ! -z "$(git add -A -n db/schema_migrations)" ]; then
+ printf "Schema version files have not been committed to the repository:\n"
+ printf "The following files should be committed:\n"
+ diff=$(git add -A -n db/schema_migrations)
+ printf "%s" "$diff"
+ exit 2
+ else
+ printf "Schema changes are correctly applied to db/structure.sql and db/schema_migrations/\n"
+ fi
}
schema_changed
diff --git a/scripts/slack b/scripts/slack
new file mode 100755
index 00000000000..60bc70a8542
--- /dev/null
+++ b/scripts/slack
@@ -0,0 +1,17 @@
+#!/bin/bash
+# This is copied from:
+# https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/bin/slack
+#
+# Sends Slack notification MSG to CI_SLACK_WEBHOOK_URL (which needs to be set).
+# ICON_EMOJI needs to be set to an icon emoji name (without the `:` around it).
+
+CHANNEL=$1
+MSG=$2
+ICON_EMOJI=$3
+
+if [ -z "$CHANNEL" ] || [ -z "$CI_SLACK_WEBHOOK_URL" ] || [ -z "$MSG" ] || [ -z "$ICON_EMOJI" ]; then
+ echo "Missing argument(s) - Use: $0 channel message icon_emoji"
+ echo "and set CI_SLACK_WEBHOOK_URL environment variable."
+else
+ curl -X POST --data-urlencode 'payload={"channel": "#'"$CHANNEL"'", "username": "GitLab QA Bot", "text": "'"$MSG"'", "icon_emoji": "'":$ICON_EMOJI:"'"}' "$CI_SLACK_WEBHOOK_URL"
+fi
diff --git a/scripts/sync-stable-branch.sh b/scripts/sync-stable-branch.sh
index 59ab52844fb..62538002278 100644
--- a/scripts/sync-stable-branch.sh
+++ b/scripts/sync-stable-branch.sh
@@ -51,7 +51,8 @@ then
echo 'Commit not available, triggering a merge train'
fi
-curl -X POST \
+curl -f \
+ -X POST \
-F token="$MERGE_TRAIN_TRIGGER_TOKEN" \
-F ref=master \
-F "variables[MERGE_FOSS]=1" \
diff --git a/scripts/trigger-build b/scripts/trigger-build
index 9f0df21e7f1..7fc550d86ee 100755
--- a/scripts/trigger-build
+++ b/scripts/trigger-build
@@ -29,10 +29,17 @@ module Trigger
end
class Base
- def invoke!(post_comment: false, downstream_job_name: nil)
+ # Can be overridden
+ def self.access_token
+ ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
+ end
+
+ def initialize
# gitlab-bot's token "GitLab multi-project pipeline polling"
- Gitlab.private_token = access_token
+ Gitlab.private_token = self.class.access_token
+ end
+ def invoke!(post_comment: false, downstream_job_name: nil)
pipeline_variables = variables
puts "Triggering downstream pipeline on #{downstream_project_path}"
@@ -74,14 +81,9 @@ module Trigger
raise NotImplementedError
end
- # Must be overridden
+ # Can be overridden
def trigger_token
- raise NotImplementedError
- end
-
- # Must be overridden
- def access_token
- raise NotImplementedError
+ ENV['CI_JOB_TOKEN']
end
# Can be overridden
@@ -133,14 +135,6 @@ module Trigger
ENV['OMNIBUS_BRANCH'] || 'master'
end
- def trigger_token
- ENV['CI_JOB_TOKEN']
- end
-
- def access_token
- ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
- end
-
def extra_variables
# Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results
# and fallback to CI_COMMIT_SHA for the `detached` pipelines.
@@ -176,10 +170,6 @@ module Trigger
ENV['BUILD_TRIGGER_TOKEN']
end
- def access_token
- ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
- end
-
def extra_variables
edition = Trigger.ee? ? 'EE' : 'CE'
@@ -205,6 +195,112 @@ module Trigger
end
end
+ class Docs < Base
+ def self.access_token
+ ENV['DOCS_API_TOKEN']
+ end
+
+ SUCCESS_MESSAGE = <<~MSG
+ => You should now be able to preview your changes under the following URL:
+
+ %<app_url>s
+
+ => For more information, see the documentation
+ => https://docs.gitlab.com/ee/development/documentation/index.html#previewing-the-changes-live
+
+ => If something doesn't work, drop a line in the #docs chat channel.
+ MSG
+
+ # Create a remote branch in gitlab-docs and immediately cancel the pipeline
+ # to avoid race conditions, since a triggered pipeline will also run right
+ # after the branch creation. This only happens the very first time a branch
+ # is created and will be skipped in subsequent runs. Read more in
+ # https://gitlab.com/gitlab-org/gitlab-docs/issues/154.
+ #
+ def deploy!
+ create_remote_branch!
+ cancel_latest_pipeline!
+ invoke!.wait!
+ display_success_message
+ end
+
+ #
+ # Remove a remote branch in gitlab-docs.
+ #
+ def cleanup!
+ Gitlab.delete_branch(downstream_project_path, ref)
+ puts "=> Remote branch '#{downstream_project_path}' deleted"
+ end
+
+ private
+
+ def downstream_project_path
+ ENV['DOCS_PROJECT_PATH'] || 'gitlab-org/gitlab-docs'
+ end
+
+ def ref
+ if ENV['CI_MERGE_REQUEST_IID'].nil?
+ "docs-preview-#{slug}-#{ENV['CI_COMMIT_REF_SLUG']}"
+ else
+ "docs-preview-#{slug}-#{ENV['CI_MERGE_REQUEST_IID']}"
+ end
+ end
+
+ def extra_variables
+ {
+ "BRANCH_#{slug.upcase}" => ENV['CI_COMMIT_REF_NAME']
+ }
+ end
+
+ def slug
+ case ENV['CI_PROJECT_PATH']
+ when 'gitlab-org/gitlab-foss'
+ 'ce'
+ when 'gitlab-org/gitlab'
+ 'ee'
+ when 'gitlab-org/gitlab-runner'
+ 'runner'
+ when 'gitlab-org/omnibus-gitlab'
+ 'omnibus'
+ when 'gitlab-org/charts/gitlab'
+ 'charts'
+ end
+ end
+
+ def app_url
+ "http://#{ref}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{slug}"
+ end
+
+ def create_remote_branch!
+ Gitlab.create_branch(downstream_project_path, ref, 'master')
+ puts "=> Remote branch '#{ref}' created"
+ rescue Gitlab::Error::BadRequest
+ puts "=> Remote branch '#{ref}' already exists!"
+ end
+
+ def cancel_latest_pipeline!
+ pipelines = nil
+
+ # Wait until the pipeline is started
+ loop do
+ sleep 1
+ puts "=> Waiting for pipeline to start..."
+ pipelines = Gitlab.pipelines(downstream_project_path, { ref: ref })
+ break if pipelines.any?
+ end
+
+ # Get the first pipeline ID which should be the only one for the branch
+ pipeline_id = pipelines.first.id
+
+ # Cancel the pipeline
+ Gitlab.cancel_pipeline(downstream_project_path, pipeline_id)
+ end
+
+ def display_success_message
+ format(SUCCESS_MESSAGE, app_url: app_url)
+ end
+ end
+
class CommitComment
def self.post!(downstream_pipeline)
Gitlab.create_commit_comment(
@@ -282,6 +378,18 @@ when 'omnibus'
Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait!
when 'cng'
Trigger::CNG.new.invoke!.wait!
+when 'docs'
+ docs_trigger = Trigger::Docs.new
+
+ case ARGV[1]
+ when 'deploy'
+ docs_trigger.deploy!
+ when 'cleanup'
+ docs_trigger.cleanup!
+ else
+ puts 'usage: trigger-build docs <deploy|cleanup>'
+ exit 1
+ end
else
puts "Please provide a valid option:
omnibus - Triggers a pipeline that builds the omnibus-gitlab package
diff --git a/scripts/trigger-build-docs b/scripts/trigger-build-docs
deleted file mode 100755
index 2957dde6fc0..00000000000
--- a/scripts/trigger-build-docs
+++ /dev/null
@@ -1,136 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'gitlab'
-
-#
-# Configure credentials to be used with gitlab gem
-#
-Gitlab.configure do |config|
- config.endpoint = 'https://gitlab.com/api/v4'
- config.private_token = ENV["DOCS_API_TOKEN"] # GitLab Docs bot access token with Developer access to gitlab-docs
-end
-
-#
-# The remote docs project
-#
-GITLAB_DOCS_REPO = 'gitlab-org/gitlab-docs'.freeze
-
-#
-# This is the branch that will be created in the gitlab-docs project.
-# Name it after the product we're previewing and the ID of the MR that
-# kicked the review app.
-#
-def docs_branch
- # Check if CI_MERGE_REQUEST_IID is present. This requires pipelines
- # for merge requests to be enabled.
- if ENV["CI_MERGE_REQUEST_IID"].nil?
- "docs-preview-#{slug}-#{ENV["CI_COMMIT_REF_SLUG"]}"
- else
- "docs-preview-#{slug}-#{ENV["CI_MERGE_REQUEST_IID"]}"
- end
-end
-
-#
-# Create a remote branch in gitlab-docs and immediately cancel the pipeline
-# to avoid race conditions, since a triggered pipeline will also run right
-# after the branch creation. This only happens the very first time a branch
-# is created and will be skipped in subsequent runs. Read more in
-# https://gitlab.com/gitlab-org/gitlab-docs/issues/154.
-#
-def create_remote_branch
- Gitlab.create_branch(GITLAB_DOCS_REPO, docs_branch, 'master')
- puts "=> Remote branch '#{docs_branch}' created"
-
- pipelines = nil
-
- # Wait until the pipeline is started
- loop do
- sleep 1
- puts "=> Waiting for pipeline to start..."
- pipelines = Gitlab.pipelines(GITLAB_DOCS_REPO, { ref: docs_branch })
- break if pipelines.any?
- end
-
- # Get the first pipeline ID which should be the only one for the branch
- pipeline_id = pipelines.first.id
-
- # Cancel the pipeline
- Gitlab.cancel_pipeline(GITLAB_DOCS_REPO, pipeline_id)
-rescue Gitlab::Error::BadRequest
- puts "=> Remote branch '#{docs_branch}' already exists"
-end
-
-#
-# Remove a remote branch in gitlab-docs
-#
-def remove_remote_branch
- Gitlab.delete_branch(GITLAB_DOCS_REPO, docs_branch)
- puts "=> Remote branch '#{docs_branch}' deleted"
-end
-
-#
-# Define suffix in review app URL based on project
-#
-def slug
- case ENV["CI_PROJECT_PATH"]
- when 'gitlab-org/gitlab-foss'
- 'ce'
- when 'gitlab-org/gitlab'
- 'ee'
- when 'gitlab-org/gitlab-runner'
- 'runner'
- when 'gitlab-org/omnibus-gitlab'
- 'omnibus'
- when 'gitlab-org/charts/gitlab'
- 'charts'
- end
-end
-
-#
-# Overriding vars in https://gitlab.com/gitlab-org/gitlab-docs/blob/master/.gitlab-ci.yml
-#
-def param_name
- "BRANCH_#{slug.upcase}"
-end
-
-#
-# Trigger a pipeline in gitlab-docs
-#
-def trigger_pipeline
- # The review app URL
- app_url = "http://#{docs_branch}.#{ENV["DOCS_REVIEW_APPS_DOMAIN"]}/#{slug}"
-
- # Create the cross project pipeline using CI_JOB_TOKEN
- pipeline = Gitlab.run_trigger(GITLAB_DOCS_REPO, ENV["CI_JOB_TOKEN"], docs_branch, { param_name => ENV["CI_COMMIT_REF_NAME"] })
-
- puts "=> Follow the status of the triggered pipeline:"
- puts ""
- puts pipeline.web_url
- puts ""
- puts "=> In a few minutes, you will be able to preview your changes under the following URL:"
- puts ""
- puts app_url
- puts ""
- puts "=> For more information, see the documentation"
- puts "=> https://docs.gitlab.com/ee/development/documentation/index.html#previewing-the-changes-live"
- puts ""
- puts "=> If something doesn't work, drop a line in the #docs chat channel."
- puts ""
-end
-
-#
-# When the first argument is deploy then create the branch and trigger pipeline
-# When it is 'stop', it deleted the remote branch. That way, we ensure there
-# are no stale remote branches and the Review server doesn't fill.
-#
-case ARGV[0]
-when 'deploy'
- create_remote_branch
- trigger_pipeline
-when 'cleanup'
- remove_remote_branch
-else
- puts "Please provide a valid option:
- deploy - Creates the remote branch and triggers a pipeline
- cleanup - Deletes the remote branch and stops the Review App"
-end