summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2019-09-13 13:26:31 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2019-09-13 13:26:31 +0000
commitb7dfe2ae4054aa40e15182fd3c6cb7dd39f131db (patch)
tree5ab080ca9cadeb6cd9578bf301e4e9e8810bed9e /scripts
parent25cb337cf12438169f1b14bc5dace8a06a7356e3 (diff)
downloadgitlab-ce-b7dfe2ae4054aa40e15182fd3c6cb7dd39f131db.tar.gz
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/build_assets_image2
-rw-r--r--scripts/ee_specific_check/ee_specific_check.rb470
-rwxr-xr-xscripts/prepare_postgres_fdw.sh15
-rwxr-xr-xscripts/review_apps/review-apps.sh2
-rwxr-xr-x[-rw-r--r--]scripts/schema_changed.sh0
5 files changed, 488 insertions, 1 deletions
diff --git a/scripts/build_assets_image b/scripts/build_assets_image
index 25b6060b6c4..aff6cb24a1b 100755
--- a/scripts/build_assets_image
+++ b/scripts/build_assets_image
@@ -8,7 +8,7 @@ fi
# Generate the image name based on the project this is being run in
ASSETS_IMAGE_NAME="gitlab-assets-ce"
-if [[ "${CI_PROJECT_NAME}" == "gitlab-ee" ]]
+if [[ "${CI_PROJECT_NAME}" == "gitlab" ]]
then
ASSETS_IMAGE_NAME="gitlab-assets-ee"
fi
diff --git a/scripts/ee_specific_check/ee_specific_check.rb b/scripts/ee_specific_check/ee_specific_check.rb
new file mode 100644
index 00000000000..98a4038ee09
--- /dev/null
+++ b/scripts/ee_specific_check/ee_specific_check.rb
@@ -0,0 +1,470 @@
+# frozen_string_literal: true
+# rubocop: disable CodeReuse/ActiveRecord
+
+module EESpecificCheck
+ WHITELIST = [
+ 'CHANGELOG-EE.md',
+ 'scripts/**/*',
+ 'vendor/assets/javascripts/jasmine-jquery.js',
+ '.gitlab-ci.yml',
+ '.gitlab/ci/rails.gitlab-ci.yml',
+ 'db/schema.rb',
+ 'locale/gitlab.pot'
+ ].freeze
+
+ CompareBase = Struct.new(:ce_base, :ee_base, :ce_head, :ee_head)
+ GitStatus = Struct.new(:porcelain, :head)
+
+ module_function
+
+ def git_version
+ say run_git_command('--version')
+ end
+
+ def say(message)
+ warn "\n#{message}", "\n" # puts would eat trailing newline
+ end
+
+ def find_compare_base
+ git_clean
+
+ ce_fetch_head = fetch_remote_ce_branch
+ ee_fetch_head = head_commit_sha
+ ce_fetch_base = find_merge_base('canonical-ce/master', ce_fetch_head)
+ ee_fetch_base = find_merge_base('canonical-ee/master', 'HEAD')
+ ce_merge_base = find_merge_base(ce_fetch_head, ee_fetch_head)
+
+ ce_updated_head =
+ find_ce_compare_head(ce_fetch_head, ce_fetch_base, ce_merge_base)
+
+ CompareBase.new(
+ ce_merge_base, ee_fetch_base, ce_updated_head, ee_fetch_head)
+ end
+
+ def setup_canonical_remotes
+ run_git_command(
+ "remote add canonical-ee https://gitlab.com/gitlab-org/gitlab-ee.git",
+ "remote add canonical-ce https://gitlab.com/gitlab-org/gitlab-ce.git",
+ "fetch canonical-ee master --quiet --depth=9999",
+ "fetch canonical-ce master --quiet --depth=9999")
+ end
+
+ def fetch_remote_ce_branch
+ setup_canonical_remotes
+
+ remote_to_fetch, branch_to_fetch = find_remote_ce_branch
+
+ run_git_command("fetch #{remote_to_fetch} #{branch_to_fetch} --quiet --depth=9999")
+
+ "#{remote_to_fetch}/#{branch_to_fetch}"
+ end
+
+ def find_merge_base(left, right)
+ merge_base = run_git_command("merge-base #{left} #{right}")
+
+ return merge_base unless merge_base.empty?
+
+ say <<~MESSAGE
+ 💥 Unfortunately we cannot find the merge-base for #{left} and #{right},
+ 💥 and we'll try to fix that in:
+ https://gitlab.com/gitlab-org/gitlab-ee/issues/9120
+
+ 💥 Before that, please run this job locally as a workaround:
+
+ ./scripts/ee-specific-lines-check
+
+ 💥 And paste the result as a discussion to show it to the maintainer.
+ 💥 If you have any questions, please ping @godfat to investigate and
+ 💥 clarify.
+ MESSAGE
+
+ exit(253)
+ end
+
+ def find_ce_compare_head(ce_fetch_head, ce_fetch_base, ce_merge_base)
+ if git_ancestor?(ce_merge_base, ce_fetch_base)
+ say("CE is ahead of EE, finding backward CE head")
+ find_backward_ce_head(ce_fetch_head, ce_fetch_base, ce_merge_base)
+ else
+ say("CE is behind of EE, finding forward CE head")
+ find_forward_ce_head(ce_merge_base, ce_fetch_head)
+ end
+ end
+
+ def git_ancestor?(ancestor, descendant)
+ run_git_command(
+ "merge-base --is-ancestor #{ancestor} #{descendant} && echo y") == 'y'
+ end
+
+ def find_backward_ce_head(ce_fetch_head, ce_fetch_base, ce_merge_base)
+ if ce_fetch_head.start_with?('canonical-ce') # No specific CE branch
+ say("No CE branch found, using merge base directly")
+ ce_merge_base
+ elsif ce_fetch_base == ce_merge_base # Up-to-date, no rebase needed
+ say("EE is up-to-date with CE, using #{ce_fetch_head} directly")
+ ce_fetch_head
+ else
+ say("Performing rebase to remove commits in CE haven't merged into EE")
+ checkout_and_rebase(ce_merge_base, ce_fetch_base, ce_fetch_head)
+ end
+ end
+
+ def find_forward_ce_head(ce_merge_base, ce_fetch_head)
+ say("Performing merge with CE master for CE branch #{ce_fetch_head}")
+ with_detached_head(ce_fetch_head) do
+ run_git_command("merge #{ce_merge_base} -s recursive -X patience -m 'ee-specific-auto-merge'")
+
+ status = git_status
+
+ if status.porcelain == ''
+ status.head
+ else
+ diff = run_git_command("diff")
+ run_git_command("merge --abort")
+
+ say <<~MESSAGE
+ 💥 Git status not clean! This means there's a conflict in
+ 💥 #{ce_fetch_head} with canonical-ce/master. Please resolve
+ 💥 the conflict from CE master and retry this job.
+
+ ⚠️ Git diff:
+
+ #{diff}
+ MESSAGE
+
+ exit(254)
+ end
+ end
+ end
+
+ # We rebase onto the commit which is the latest commit presented in both
+ # CE and EE, i.e. ce_merge_base, cutting off commits aren't merged into
+ # EE yet. Here's an example:
+ #
+ # * o: Relevant commits
+ # * x: Irrelevant commits
+ # * !: Commits we want to cut off from CE branch
+ #
+ # ^-> o CE branch (ce_fetch_head)
+ # / (ce_fetch_base)
+ # o -> o -> ! -> x CE master
+ # v (ce_merge_base)
+ # o -> o -> o -> x EE master
+ # \ (ee_fetch_base)
+ # v-> o EE branch
+ #
+ # We want to rebase above into this: (we only change the connection)
+ #
+ # -> - -> o CE branch (ce_fetch_head)
+ # / (ce_fetch_base)
+ # o -> o -> ! -> x CE master
+ # v (ce_merge_base)
+ # o -> o -> o -> x EE master
+ # \ (ee_fetch_base)
+ # v-> o EE branch
+ #
+ # Therefore we rebase onto ce_merge_base, which is based off CE master,
+ # for the CE branch (ce_fetch_head), effective remove the commit marked
+ # as ! in the graph for CE branch. We need to remove it because it's not
+ # merged into EE yet, therefore won't be available in the EE branch.
+ #
+ # After rebase is done, then we could compare against
+ # ce_merge_base..ee_fetch_base along with ce_fetch_head..HEAD (EE branch)
+ # where ce_merge_base..ee_fetch_base is the update-to-date
+ # CE/EE difference and ce_fetch_head..HEAD is the changes we made in
+ # CE and EE branches.
+ def checkout_and_rebase(new_base, old_base, target_head)
+ with_detached_head(target_head) do
+ run_git_command("rebase --onto #{new_base} #{old_base} #{target_head}")
+
+ status = git_status
+
+ if status.porcelain == ''
+ status.head
+ else
+ diff = run_git_command("diff")
+ run_git_command("rebase --abort")
+
+ say <<~MESSAGE
+ 💥 Git status is not clean! This means the CE branch has or had a
+ 💥 conflict with CE master, and we cannot resolve this in an
+ 💥 automatic way.
+ 💥
+ 💥 Please rebase #{target_head} with CE master.
+ 💥
+ 💥 For more details, please read:
+ 💥 https://gitlab.com/gitlab-org/gitlab-ee/issues/6038#note_86862115
+ 💥
+ 💥 Git diff:
+
+ #{diff}
+ MESSAGE
+
+ exit(255)
+ end
+ end
+ end
+
+ def with_detached_head(target_head)
+ # So that we could switch back. CI sometimes doesn't have the branch,
+ # so we don't use current_branch here
+ head = current_head
+
+ # Use detached HEAD so that we don't update HEAD
+ run_git_command("checkout -f #{target_head}")
+ git_clean
+
+ yield
+ ensure # ensure would still run if we call exit, don't worry
+ # Make sure to switch back
+ run_git_command("checkout -f #{head}")
+ git_clean
+ end
+
+ def head_commit_sha
+ run_git_command("rev-parse HEAD")
+ end
+
+ def git_status
+ GitStatus.new(
+ run_git_command("status --porcelain"),
+ head_commit_sha
+ )
+ end
+
+ def git_clean
+ # We're still seeing errors not ignoring knapsack/ and rspec_flaky/
+ # Instead of waiting that populate over all the branches, we could
+ # just remove untracked files anyway, only on CI of course in case
+ # we're wiping people's data!
+ # See https://gitlab.com/gitlab-org/gitlab-ee/issues/5912
+ # Also see https://gitlab.com/gitlab-org/gitlab-ee/-/jobs/68194333
+ run_git_command('clean -fd') if ENV['CI']
+ end
+
+ def remove_remotes
+ run_git_command(
+ "remote remove canonical-ee",
+ "remote remove canonical-ce",
+ "remote remove target-ce")
+ end
+
+ def updated_diff_numstat(from, to)
+ scan_diff_numstat(
+ run_git_command("diff #{from}..#{to} --numstat -- . ':!ee' ':!qa/qa/ee' ':!qa/qa/ee.rb' ':!qa/qa/specs/features/ee'"))
+ end
+
+ def find_remote_ce_branch
+ branch_to_fetch = matching_ce_refs.first
+
+ if branch_to_fetch
+ say "💪 We found the branch '#{branch_to_fetch}' in the #{ce_repo_url} repository. We will fetch it."
+
+ run_git_command("remote add target-ce #{ce_repo_url}")
+
+ ['target-ce', branch_to_fetch]
+ else
+ say <<~MESSAGE
+ ⚠️ We did not find a branch that would match the current '#{current_branch}' branch in the #{ce_repo_url} repository. We will fetch 'master' instead.
+ ℹ️ If you have a CE branch for the current branch, make sure that its name includes '#{minimal_ce_branch_name}'.
+ MESSAGE
+
+ %w[canonical-ce master]
+ end
+ end
+
+ def ce_repo_url
+ @ce_repo_url ||=
+ begin
+ repo_url = ENV.fetch('CI_REPOSITORY_URL', 'https://gitlab.com/gitlab-org/gitlab-ce.git')
+ # This workaround can be removed once we rename the dev CE project
+ # https://gitlab.com/gitlab-org/gitlab-ce/issues/59107
+ project_name = repo_url =~ /dev\.gitlab\.org/ ? 'gitlabhq' : 'gitlab-ce'
+
+ repo_url.sub('gitlab-ee', project_name)
+ end
+ end
+
+ def current_head
+ @current_head ||= ENV.fetch('CI_COMMIT_SHA', current_branch)
+ end
+
+ def current_branch
+ @current_branch ||= ENV.fetch('CI_COMMIT_REF_NAME', `git rev-parse --abbrev-ref HEAD`).strip
+ end
+
+ def minimal_ce_branch_name
+ @minimal_ce_branch_name ||= current_branch.sub(/(\Aee\-|\-ee\z)/, '')
+ end
+
+ def matching_ce_refs
+ @matching_ce_refs ||=
+ run_git_command("ls-remote #{ce_repo_url} \"*#{minimal_ce_branch_name}*\"")
+ .scan(%r{(?<=refs/heads/|refs/tags/).+})
+ .select { |branch| branch.match?(/\b#{minimal_ce_branch_name}\b/i) }
+ .sort_by(&:size)
+ end
+
+ def scan_diff_numstat(numstat)
+ numstat.scan(/(\d+)\s+(\d+)\s+(.+)/)
+ .each_with_object(Hash.new(0)) do |(added, deleted, file), result|
+ result[file] = added.to_i + deleted.to_i
+ end
+ end
+
+ def run_git_command(*commands)
+ cmds = commands.map { |cmd| "git #{cmd}" }
+
+ output = run_command(*cmds)
+
+ if commands.size == 1
+ output.first
+ else
+ output
+ end
+ end
+
+ def run_command(*commands)
+ commands.map do |cmd|
+ warn "=> Running `#{cmd}`"
+
+ `#{cmd}`.strip
+ end
+ end
+end
+
+if $0 == __FILE__
+ require 'rspec/autorun'
+
+ RSpec.describe EESpecificCheck do
+ subject { Class.new { include EESpecificCheck }.new }
+
+ before do
+ allow(subject).to receive(:warn)
+
+ EESpecificCheck.private_instance_methods.each do |name|
+ subject.class.__send__(:public, name) # rubocop:disable GitlabSecurity/PublicSend
+ end
+ end
+
+ describe '.run_git_command' do
+ it 'returns the single output when there is a single command' do
+ output = subject.run_git_command('status')
+
+ expect(output).to be_kind_of(String)
+ expect(subject).to have_received(:warn).with(/git status/)
+ end
+
+ it 'returns an array of output for more commands' do
+ output = subject.run_git_command('status', 'help')
+
+ expect(output).to all(be_a(String))
+ expect(subject).to have_received(:warn).with(/git status/)
+ expect(subject).to have_received(:warn).with(/git help/)
+ end
+ end
+
+ describe '.find_merge_base' do
+ context 'when it cannot find the merge base' do
+ before do
+ allow(subject).to receive(:say)
+ allow(subject).to receive(:exit)
+
+ expect(subject).to receive(:run_git_command).and_return('')
+ end
+
+ it 'calls exit(253) to fail the job and ask run it locally' do
+ subject.find_merge_base('master', 'HEAD')
+
+ expect(subject).to have_received(:say)
+ .with(Regexp.union('./scripts/ee-specific-lines-check'))
+ expect(subject).to have_received(:exit)
+ .with(253)
+ end
+ end
+
+ context 'when it found the merge base' do
+ before do
+ expect(subject).to receive(:run_git_command).and_return('deadbeef')
+ end
+
+ it 'returns the found merge base' do
+ output = subject.find_merge_base('master', 'HEAD')
+
+ expect(output).to eq('deadbeef')
+ end
+ end
+ end
+
+ describe '.matching_ce_refs' do
+ before do
+ expect(subject).to receive(:current_branch).and_return(ee_branch)
+ expect(subject).to receive(:run_git_command)
+ .and_return(ls_remote_output)
+ end
+
+ describe 'simple cases' do
+ let(:ls_remote_output) do
+ <<~OUTPUT
+ d6602ec5194c87b0fc87103ca4d67251c76f233a\trefs/tags/v9
+ f25a265a342aed6041ab0cc484224d9ca54b6f41\trefs/tags/v9.12
+ c5db5456ae3b0873fc659c19fafdde22313cc441\trefs/tags/v9.123
+ 0918385dbd9656cab0d1d81ba7453d49bbc16250\trefs/heads/v9.x
+ 28862662b749fe981386814e2dba87b0e72c1eab\trefs/remotes/remote_mirror_3059/v9-to-fix-http-case-problems
+ 5e3496802098c86050c5b463507f3a68a83a9f02\trefs/remotes/remote_mirror_3059/29036-use-slack-service-v9
+ OUTPUT
+ end
+
+ context 'with a ee- prefix' do
+ let(:ee_branch) { 'ee-v9' }
+
+ it 'sorts by matching size' do
+ expect(subject.matching_ce_refs).to eq(%w[v9 v9.x v9.12 v9.123])
+ end
+ end
+
+ context 'with a -ee suffix' do
+ let(:ee_branch) { 'v9-ee' }
+
+ it 'sorts by matching size' do
+ expect(subject.matching_ce_refs).to eq(%w[v9 v9.x v9.12 v9.123])
+ end
+ end
+ end
+
+ describe 'with ambiguous branch name' do
+ let(:ls_remote_output) do
+ <<~OUTPUT
+ 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/feature/sm/35954-expand-kubernetesservice-to-use-username-password
+ 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ce-to-ee-231
+ 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ce-to-ee-2
+ 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ce-to-1
+ 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ee-to-ce-123
+ 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ee-to-ce-12
+ 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/to-ce-1
+ 28862662b749fe981386814e2dba87b0e72c1eab\trefs/remotes/remote_mirror_3059/27056-upgrade-vue-resource-to-1-0-3-to-fix-http-case-problems
+ 5e3496802098c86050c5b463507f3a68a83a9f02\trefs/remotes/remote_mirror_3059/29036-use-slack-service-to-notify-of-failed-pipelines
+ OUTPUT
+ end
+
+ context 'with a ee- prefix' do
+ let(:ee_branch) { 'ee-to-ce' }
+ let(:minimal_ce_branch) { 'to-ce' }
+
+ it 'sorts by matching size' do
+ expect(subject.matching_ce_refs).to eq(%w[to-ce-1 ee-to-ce-12 ee-to-ce-123])
+ end
+ end
+
+ context 'with a -ee suffix' do
+ let(:ee_branch) { 'ce-to-ee' }
+ let(:minimal_ce_branch) { 'ce-to' }
+
+ it 'sorts by matching size' do
+ expect(subject.matching_ce_refs).to eq(%w[ce-to-1 ce-to-ee-2 ce-to-ee-231])
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/scripts/prepare_postgres_fdw.sh b/scripts/prepare_postgres_fdw.sh
new file mode 100755
index 00000000000..69442d2881b
--- /dev/null
+++ b/scripts/prepare_postgres_fdw.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+psql -h postgres -U postgres gitlabhq_geo_test <<EOF
+CREATE EXTENSION postgres_fdw;
+CREATE SERVER gitlab_secondary FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host 'localhost', dbname 'gitlabhq_test');
+CREATE USER MAPPING FOR current_user SERVER gitlab_secondary OPTIONS (user 'postgres', password '');
+CREATE SCHEMA gitlab_secondary;
+IMPORT FOREIGN SCHEMA public FROM SERVER gitlab_secondary INTO gitlab_secondary;
+GRANT USAGE ON FOREIGN SERVER gitlab_secondary TO current_user;
+EOF
+
+# Ensure the FDW setting is enabled
+sed -i '/fdw:/d' config/database_geo.yml
+sed -i '/gitlabhq_geo_test/a\
+\ \ fdw: true' config/database_geo.yml
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index 76eb67b1a2e..8b64dd3292c 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -255,6 +255,8 @@ EOF
}
function display_deployment_debug() {
+ kubectl get pods -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG}
+
migrations_pod=$(get_pod "migrations");
if [ -z "${migrations_pod}" ]; then
echoerr "Migrations pod not found."
diff --git a/scripts/schema_changed.sh b/scripts/schema_changed.sh
index b5e510c2367..b5e510c2367 100644..100755
--- a/scripts/schema_changed.sh
+++ b/scripts/schema_changed.sh