path: root/scripts/ee_specific_check/ee_specific_check.rb

# frozen_string_literal: true
# rubocop: disable CodeReuse/ActiveRecord

module EESpecificCheck
  WHITELIST = [
    'CHANGELOG-EE.md',
    'scripts/**/*',
    'vendor/assets/javascripts/jasmine-jquery.js',
    '.gitlab-ci.yml',
    '.gitlab/ci/rails.gitlab-ci.yml',
    'db/schema.rb',
    'locale/gitlab.pot'
  ].freeze

  CompareBase = Struct.new(:ce_base, :ee_base, :ce_head, :ee_head)
  GitStatus = Struct.new(:porcelain, :head)

  module_function

  def git_version
    say run_git_command('--version')
  end

  def say(message)
    warn "\n#{message}", "\n" # puts would eat trailing newline
  end

  def find_compare_base
    git_clean

    ce_fetch_head = fetch_remote_ce_branch
    ee_fetch_head = head_commit_sha
    ce_fetch_base = find_merge_base('canonical-ce/master', ce_fetch_head)
    ee_fetch_base = find_merge_base('canonical-ee/master', 'HEAD')
    ce_merge_base = find_merge_base(ce_fetch_head, ee_fetch_head)

    ce_updated_head =
      find_ce_compare_head(ce_fetch_head, ce_fetch_base, ce_merge_base)

    CompareBase.new(
      ce_merge_base, ee_fetch_base, ce_updated_head, ee_fetch_head)
  end

  def setup_canonical_remotes
    run_git_command(
      "remote add canonical-ee https://gitlab.com/gitlab-org/gitlab.git",
      "remote add canonical-ce https://gitlab.com/gitlab-org/gitlab-foss.git",
      "fetch canonical-ee master --quiet --depth=9999",
      "fetch canonical-ce master --quiet --depth=9999")
  end

  def fetch_remote_ce_branch
    setup_canonical_remotes

    remote_to_fetch, branch_to_fetch = find_remote_ce_branch

    run_git_command("fetch #{remote_to_fetch} #{branch_to_fetch} --quiet --depth=9999")

    "#{remote_to_fetch}/#{branch_to_fetch}"
  end

  def find_merge_base(left, right)
    merge_base = run_git_command("merge-base #{left} #{right}")

    return merge_base unless merge_base.empty?

    say <<~MESSAGE
      💥 Unfortunately we cannot find the merge-base for #{left} and #{right},
      💥 and we'll try to fix that in:
          https://gitlab.com/gitlab-org/gitlab/issues/9120

      💥 Before that, please run this job locally as a workaround:

          ./scripts/ee-specific-lines-check

      💥 And paste the result as a discussion to show it to the maintainer.
      💥 If you have any questions, please ping @godfat to investigate and
      💥 clarify.
    MESSAGE

    exit(253)
  end

  def find_ce_compare_head(ce_fetch_head, ce_fetch_base, ce_merge_base)
    if git_ancestor?(ce_merge_base, ce_fetch_base)
      say("CE is ahead of EE, finding backward CE head")
      find_backward_ce_head(ce_fetch_head, ce_fetch_base, ce_merge_base)
    else
      say("CE is behind of EE, finding forward CE head")
      find_forward_ce_head(ce_merge_base, ce_fetch_head)
    end
  end

  def git_ancestor?(ancestor, descendant)
    run_git_command(
      "merge-base --is-ancestor #{ancestor} #{descendant} && echo y") == 'y'
  end

  def find_backward_ce_head(ce_fetch_head, ce_fetch_base, ce_merge_base)
    if ce_fetch_head.start_with?('canonical-ce') # No specific CE branch
      say("No CE branch found, using merge base directly")
      ce_merge_base
    elsif ce_fetch_base == ce_merge_base # Up-to-date, no rebase needed
      say("EE is up-to-date with CE, using #{ce_fetch_head} directly")
      ce_fetch_head
    else
      say("Performing rebase to remove commits in CE haven't merged into EE")
      checkout_and_rebase(ce_merge_base, ce_fetch_base, ce_fetch_head)
    end
  end

  def find_forward_ce_head(ce_merge_base, ce_fetch_head)
    say("Performing merge with CE master for CE branch #{ce_fetch_head}")
    with_detached_head(ce_fetch_head) do
      run_git_command("merge #{ce_merge_base} -s recursive -X patience -m 'ee-specific-auto-merge'")

      status = git_status

      if status.porcelain == ''
        status.head
      else
        diff = run_git_command("diff")
        run_git_command("merge --abort")

        say <<~MESSAGE
          💥 Git status not clean! This means there's a conflict in
          💥 #{ce_fetch_head} with canonical-ce/master. Please resolve
          💥 the conflict from CE master and retry this job.

          ⚠️ Git diff:

          #{diff}
        MESSAGE

        exit(254)
      end
    end
  end

  # We rebase onto the commit which is the latest commit presented in both
  # CE and EE, i.e. ce_merge_base, cutting off commits aren't merged into
  # EE yet. Here's an example:
  #
  # * o: Relevant commits
  # * x: Irrelevant commits
  # * !: Commits we want to cut off from CE branch
  #
  #                ^-> o CE branch (ce_fetch_head)
  #               / (ce_fetch_base)
  #     o -> o -> ! -> x CE master
  #          v (ce_merge_base)
  #     o -> o -> o -> x EE master
  #               \ (ee_fetch_base)
  #                v-> o EE branch
  #
  # We want to rebase above into this: (we only change the connection)
  #
  #            -> - -> o CE branch (ce_fetch_head)
  #           / (ce_fetch_base)
  #     o -> o -> ! -> x CE master
  #          v (ce_merge_base)
  #     o -> o -> o -> x EE master
  #               \ (ee_fetch_base)
  #                v-> o EE branch
  #
  # Therefore we rebase onto ce_merge_base, which is based off CE master,
  # for the CE branch (ce_fetch_head), effective remove the commit marked
  # as ! in the graph for CE branch. We need to remove it because it's not
  # merged into EE yet, therefore won't be available in the EE branch.
  #
  # After rebase is done, then we could compare against
  # ce_merge_base..ee_fetch_base along with ce_fetch_head..HEAD (EE branch)
  # where ce_merge_base..ee_fetch_base is the update-to-date
  # CE/EE difference and ce_fetch_head..HEAD is the changes we made in
  # CE and EE branches.
  def checkout_and_rebase(new_base, old_base, target_head)
    with_detached_head(target_head) do
      run_git_command("rebase --onto #{new_base} #{old_base} #{target_head}")

      status = git_status

      if status.porcelain == ''
        status.head
      else
        diff = run_git_command("diff")
        run_git_command("rebase --abort")

        say <<~MESSAGE
          💥 Git status is not clean! This means the CE branch has or had a
          💥 conflict with CE master, and we cannot resolve this in an
          💥 automatic way.
          💥
          💥 Please rebase #{target_head} with CE master.
          💥
          💥 For more details, please read:
          💥   https://gitlab.com/gitlab-org/gitlab/issues/6038#note_86862115
          💥
          💥 Git diff:

          #{diff}
        MESSAGE

        exit(255)
      end
    end
  end

  def with_detached_head(target_head)
    # So that we could switch back. CI sometimes doesn't have the branch,
    # so we don't use current_branch here
    head = current_head

    # Use detached HEAD so that we don't update HEAD
    run_git_command("checkout -f #{target_head}")
    git_clean

    yield
  ensure # ensure would still run if we call exit, don't worry
    # Make sure to switch back
    run_git_command("checkout -f #{head}")
    git_clean
  end

  def head_commit_sha
    run_git_command("rev-parse HEAD")
  end

  def git_status
    GitStatus.new(
      run_git_command("status --porcelain"),
      head_commit_sha
    )
  end

  def git_clean
    # We're still seeing errors not ignoring knapsack/ and rspec_flaky/
    # Instead of waiting that populate over all the branches, we could
    # just remove untracked files anyway, only on CI of course in case
    # we're wiping people's data!
    # See https://gitlab.com/gitlab-org/gitlab/issues/5912
    # Also see https://gitlab.com/gitlab-org/gitlab/-/jobs/68194333
    run_git_command('clean -fd') if ENV['CI']
  end

  def remove_remotes
    run_git_command(
      "remote remove canonical-ee",
      "remote remove canonical-ce",
      "remote remove target-ce")
  end

  def updated_diff_numstat(from, to)
    scan_diff_numstat(
      run_git_command("diff #{from}..#{to} --numstat -- . ':!ee' ':!qa/qa/ee' ':!qa/qa/ee.rb' ':!qa/qa/specs/features/ee'"))
  end

  def find_remote_ce_branch
    branch_to_fetch = matching_ce_refs.first

    if branch_to_fetch
      say "💪 We found the branch '#{branch_to_fetch}' in the #{ce_repo_url} repository. We will fetch it."

      run_git_command("remote add target-ce #{ce_repo_url}")

      ['target-ce', branch_to_fetch]
    else
      say <<~MESSAGE
        ⚠️ We did not find a branch that would match the current '#{current_branch}' branch in the #{ce_repo_url} repository. We will fetch 'master' instead.
        ℹ️ If you have a CE branch for the current branch, make sure that its name includes '#{minimal_ce_branch_name}'.
      MESSAGE

      %w[canonical-ce master]
    end
  end

  def ce_repo_url
    @ce_repo_url ||=
      begin
        repo_url = ENV.fetch('CI_REPOSITORY_URL', 'https://gitlab.com/gitlab-org/gitlab-foss.git')
        # This workaround can be removed once we rename the dev CE project
        # https://gitlab.com/gitlab-org/gitlab-foss/issues/59107
        project_name = repo_url =~ /dev\.gitlab\.org/ ? 'gitlabhq' : 'gitlab-ce'

        repo_url.sub('gitlab-ee', project_name)
      end
  end

  def current_head
    @current_head ||= ENV.fetch('CI_COMMIT_SHA', current_branch)
  end

  def current_branch
    @current_branch ||= ENV.fetch('CI_COMMIT_REF_NAME', `git rev-parse --abbrev-ref HEAD`).strip
  end

  def minimal_ce_branch_name
    @minimal_ce_branch_name ||= current_branch.sub(/(\Aee\-|\-ee\z)/, '')
  end

  def matching_ce_refs
    @matching_ce_refs ||=
      run_git_command("ls-remote #{ce_repo_url} \"*#{minimal_ce_branch_name}*\"")
        .scan(%r{(?<=refs/heads/|refs/tags/).+})
        .select { |branch| branch.match?(/\b#{minimal_ce_branch_name}\b/i) }
        .sort_by(&:size)
  end

  def scan_diff_numstat(numstat)
    numstat.scan(/(\d+)\s+(\d+)\s+(.+)/)
      .each_with_object(Hash.new(0)) do |(added, deleted, file), result|
        result[file] = added.to_i + deleted.to_i
      end
  end

  def run_git_command(*commands)
    cmds = commands.map { |cmd| "git #{cmd}" }

    output = run_command(*cmds)

    if commands.size == 1
      output.first
    else
      output
    end
  end

  def run_command(*commands)
    commands.map do |cmd|
      warn "=> Running `#{cmd}`"

      `#{cmd}`.strip
    end
  end
end

if $0 == __FILE__
  require 'rspec/autorun'

  RSpec.describe EESpecificCheck do
    subject { Class.new { include EESpecificCheck }.new }

    before do
      allow(subject).to receive(:warn)

      EESpecificCheck.private_instance_methods.each do |name|
        subject.class.__send__(:public, name) # rubocop:disable GitlabSecurity/PublicSend
      end
    end

    describe '.run_git_command' do
      it 'returns the single output when there is a single command' do
        output = subject.run_git_command('status')

        expect(output).to be_kind_of(String)
        expect(subject).to have_received(:warn).with(/git status/)
      end

      it 'returns an array of output for more commands' do
        output = subject.run_git_command('status', 'help')

        expect(output).to all(be_a(String))
        expect(subject).to have_received(:warn).with(/git status/)
        expect(subject).to have_received(:warn).with(/git help/)
      end
    end

    describe '.find_merge_base' do
      context 'when it cannot find the merge base' do
        before do
          allow(subject).to receive(:say)
          allow(subject).to receive(:exit)

          expect(subject).to receive(:run_git_command).and_return('')
        end

        it 'calls exit(253) to fail the job and ask run it locally' do
          subject.find_merge_base('master', 'HEAD')

          expect(subject).to have_received(:say)
            .with(Regexp.union('./scripts/ee-specific-lines-check'))
          expect(subject).to have_received(:exit)
            .with(253)
        end
      end

      context 'when it found the merge base' do
        before do
          expect(subject).to receive(:run_git_command).and_return('deadbeef')
        end

        it 'returns the found merge base' do
          output = subject.find_merge_base('master', 'HEAD')

          expect(output).to eq('deadbeef')
        end
      end
    end

    describe '.matching_ce_refs' do
      before do
        expect(subject).to receive(:current_branch).and_return(ee_branch)
        expect(subject).to receive(:run_git_command)
          .and_return(ls_remote_output)
      end

      describe 'simple cases' do
        let(:ls_remote_output) do
          <<~OUTPUT
          d6602ec5194c87b0fc87103ca4d67251c76f233a\trefs/tags/v9
          f25a265a342aed6041ab0cc484224d9ca54b6f41\trefs/tags/v9.12
          c5db5456ae3b0873fc659c19fafdde22313cc441\trefs/tags/v9.123
          0918385dbd9656cab0d1d81ba7453d49bbc16250\trefs/heads/v9.x
          28862662b749fe981386814e2dba87b0e72c1eab\trefs/remotes/remote_mirror_3059/v9-to-fix-http-case-problems
          5e3496802098c86050c5b463507f3a68a83a9f02\trefs/remotes/remote_mirror_3059/29036-use-slack-service-v9
          OUTPUT
        end

        context 'with a ee- prefix' do
          let(:ee_branch) { 'ee-v9' }

          it 'sorts by matching size' do
            expect(subject.matching_ce_refs).to eq(%w[v9 v9.x v9.12 v9.123])
          end
        end

        context 'with a -ee suffix' do
          let(:ee_branch) { 'v9-ee' }

          it 'sorts by matching size' do
            expect(subject.matching_ce_refs).to eq(%w[v9 v9.x v9.12 v9.123])
          end
        end
      end

      describe 'with ambiguous branch name' do
        let(:ls_remote_output) do
          <<~OUTPUT
          954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/feature/sm/35954-expand-kubernetesservice-to-use-username-password
          954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ce-to-ee-231
          954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ce-to-ee-2
          954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ce-to-1
          954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ee-to-ce-123
          954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ee-to-ce-12
          954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/to-ce-1
          28862662b749fe981386814e2dba87b0e72c1eab\trefs/remotes/remote_mirror_3059/27056-upgrade-vue-resource-to-1-0-3-to-fix-http-case-problems
          5e3496802098c86050c5b463507f3a68a83a9f02\trefs/remotes/remote_mirror_3059/29036-use-slack-service-to-notify-of-failed-pipelines
          OUTPUT
        end

        context 'with a ee- prefix' do
          let(:ee_branch) { 'ee-to-ce' }
          let(:minimal_ce_branch) { 'to-ce' }

          it 'sorts by matching size' do
            expect(subject.matching_ce_refs).to eq(%w[to-ce-1 ee-to-ce-12 ee-to-ce-123])
          end
        end

        context 'with a -ee suffix' do
          let(:ee_branch) { 'ce-to-ee' }
          let(:minimal_ce_branch) { 'ce-to' }

          it 'sorts by matching size' do
            expect(subject.matching_ce_refs).to eq(%w[ce-to-1 ce-to-ee-2 ce-to-ee-231])
          end
        end
      end
    end
  end
end