Improve the ee_compat_check_job to handle gitlab-ee forks

Signed-off-by: Rémy Coutable <remy@rymai.me>

1 files changed, 91 insertions, 57 deletions

diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb
index 0fb71976883..5fdd5dcd374 100644
--- a/lib/gitlab/ee_compat_check.rb
+++ b/lib/gitlab/ee_compat_check.rb
@@ -2,8 +2,8 @@
 module Gitlab
   # Checks if a set of migrations requires downtime or not.
   class EeCompatCheck
-    DEFAULT_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze
-    EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze
+    CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze
+    CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze
     CHECK_DIR = Rails.root.join('ee_compat_check')
     IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb}i.freeze
     PLEASE_READ_THIS_BANNER = %Q{
@@ -11,57 +11,81 @@ module Gitlab
       ===================== PLEASE READ THIS =====================
       ============================================================
     }.freeze
+    STAY_STRONG_LINK_TO_DOCS = %Q{
+      Stay 💪! For more information, see
+      https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html
+    }.freeze
     THANKS_FOR_READING_BANNER = %Q{
       ============================================================
       ==================== THANKS FOR READING ====================
       ============================================================\n
     }.freeze
 
-    attr_reader :ee_repo_dir, :patches_dir, :ce_project_url, :ce_repo_url, :ce_branch, :ee_branch_found
+    attr_reader :ee_repo_dir, :patches_dir
+    attr_reader :ce_project_url, :ee_repo_url
+    attr_reader :ce_branch, :ee_remote_with_branch, :ee_branch_found
     attr_reader :job_id, :failed_files
 
-    def initialize(branch:, ce_project_url: DEFAULT_CE_PROJECT_URL, job_id: nil)
+    def initialize(branch:, ce_project_url: CANONICAL_CE_PROJECT_URL, job_id: nil)
       @ee_repo_dir = CHECK_DIR.join('ee-repo')
       @patches_dir = CHECK_DIR.join('patches')
       @ce_branch = branch
       @ce_project_url = ce_project_url
-      @ce_repo_url = "#{ce_project_url}.git"
+      @ee_repo_url = ce_public_repo_url.sub('gitlab-ce', 'gitlab-ee')
       @job_id = job_id
     end
 
     def check
       ensure_patches_dir
-      add_remote('canonical-ce', "#{DEFAULT_CE_PROJECT_URL}.git")
-      generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, remote: 'canonical-ce')
+      # We're generating the patch against the canonical-ce remote since forks'
+      # master branch are not necessarily up-to-date.
+      add_remote('canonical-ce', "#{CANONICAL_CE_PROJECT_URL}.git")
+      generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, branch_remote: 'origin', master_remote: 'canonical-ce')
 
       ensure_ee_repo
       Dir.chdir(ee_repo_dir) do
         step("In the #{ee_repo_dir} directory")
 
-        add_remote('canonical-ee', EE_REPO_URL)
+        ee_remotes.each do |key, url|
+          add_remote(key, url)
+        end
+        fetch(branch: 'master', depth: 20, remote: 'canonical-ee')
 
         status = catch(:halt_check) do
           ce_branch_compat_check!
           delete_ee_branches_locally!
           ee_branch_presence_check!
 
-          step("Checking out #{ee_branch_found}", %W[git checkout -b #{ee_branch_found} canonical-ee/#{ee_branch_found}])
-          generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, remote: 'canonical-ee')
+          step("Checking out #{ee_remote_with_branch}/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} #{ee_remote_with_branch}/#{ee_branch_found}])
+          generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, branch_remote: ee_remote_with_branch, master_remote: 'canonical-ee')
           ee_branch_compat_check!
         end
 
         delete_ee_branches_locally!
 
-        if status.nil?
-          true
-        else
-          false
-        end
+        status.nil?
       end
     end
 
     private
 
+    def fork?
+      ce_project_url != CANONICAL_CE_PROJECT_URL
+    end
+
+    def ee_remotes
+      return @ee_remotes if defined?(@ee_remotes)
+
+      remotes =
+        {
+          'ee' => ee_repo_url,
+          'canonical-ee' => CANONICAL_EE_REPO_URL
+        }
+      remotes.delete('ee') unless fork?
+
+      @ee_remotes = remotes
+    end
+
     def add_remote(name, url)
       step(
         "Adding the #{name} remote (#{url})",
@@ -70,28 +94,32 @@ module Gitlab
     end
 
     def ensure_ee_repo
-      if Dir.exist?(ee_repo_dir)
-        step("#{ee_repo_dir} already exists")
-      else
-        step(
-          "Cloning #{EE_REPO_URL} into #{ee_repo_dir}",
-          %W[git clone --branch master --single-branch --depth=200 #{EE_REPO_URL} #{ee_repo_dir}]
-        )
+      unless clone_repo(ee_repo_url, ee_repo_dir)
+        # Fallback to using the canonical EE if there is no forked EE
+        clone_repo(CANONICAL_EE_REPO_URL, ee_repo_dir)
       end
     end
 
+    def clone_repo(url, dir)
+      _, status = step(
+        "Cloning #{url} into #{dir}",
+        %W[git clone --branch master --single-branch --depth=200 #{url} #{dir}]
+      )
+      status.zero?
+    end
+
     def ensure_patches_dir
       FileUtils.mkdir_p(patches_dir)
     end
 
-    def generate_patch(branch:, patch_path:, remote:)
+    def generate_patch(branch:, patch_path:, branch_remote:, master_remote:)
       FileUtils.rm(patch_path, force: true)
 
-      find_merge_base_with_master(branch: branch, master_remote: remote)
+      find_merge_base_with_master(branch: branch, branch_remote: branch_remote, master_remote: master_remote)
 
       step(
-        "Generating the patch against #{remote}/master in #{patch_path}",
-        %W[git diff --binary #{remote}/master...origin/#{branch}]
+        "Generating the patch against #{master_remote}/master in #{patch_path}",
+        %W[git diff --binary #{master_remote}/master...#{branch_remote}/#{branch}]
       ) do |output, status|
         throw(:halt_check, :ko) unless status.zero?
 
@@ -109,23 +137,22 @@ module Gitlab
     end
 
     def ee_branch_presence_check!
-      _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch canonical-ee #{ee_branch_prefix}])
-
-      if status.zero?
-        @ee_branch_found = ee_branch_prefix
-        return
+      ee_remotes.keys.each do |remote|
+        [ee_branch_prefix, ee_branch_suffix].each do |branch|
+          _, status = step("Fetching #{remote}/#{ee_branch_prefix}", %W[git fetch #{remote} #{branch}])
+
+          if status.zero?
+            @ee_remote_with_branch = remote
+            @ee_branch_found = branch
+            return true
+          end
+        end
       end
 
-      _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch canonical-ee #{ee_branch_suffix}])
-
-      if status.zero?
-        @ee_branch_found = ee_branch_suffix
-      else
-        puts
-        puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
+      puts
+      puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
 
-        throw(:halt_check, :ko)
-      end
+      throw(:halt_check, :ko)
     end
 
     def ee_branch_compat_check!
@@ -181,10 +208,10 @@ module Gitlab
       command(%W[git branch --delete --force #{ee_branch_suffix}])
     end
 
-    def merge_base_found?(master_remote:, branch:)
+    def merge_base_found?(branch:, branch_remote:, master_remote:)
       step(
         "Finding merge base with #{master_remote}/master",
-        %W[git merge-base #{master_remote}/master origin/#{branch}]
+        %W[git merge-base #{master_remote}/master #{branch_remote}/#{branch}]
       ) do |output, status|
         if status.zero?
           puts "Merge base was found: #{output}"
@@ -193,7 +220,7 @@ module Gitlab
       end
     end
 
-    def find_merge_base_with_master(branch:, master_remote:)
+    def find_merge_base_with_master(branch:, branch_remote:, master_remote:)
       # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403)
       # In total we go (20 + 54 + 148 + 403 = 625) commits deeper
       depth = 20
@@ -202,10 +229,10 @@ module Gitlab
           depth += Math.exp(factor).to_i
           # Repository is initially cloned with a depth of 20 so we need to fetch
           # deeper in the case the branch has more than 20 commits on top of master
-          fetch(branch: branch, depth: depth, remote: 'origin')
+          fetch(branch: branch, depth: depth, remote: branch_remote)
           fetch(branch: 'master', depth: depth, remote: master_remote)
 
-          merge_base_found?(master_remote: master_remote, branch: branch)
+          merge_base_found?(branch: branch, branch_remote: branch_remote, master_remote: master_remote)
         end
 
       raise "\n#{branch} is too far behind #{master_remote}/master, please rebase it!\n" unless success
@@ -274,6 +301,13 @@ module Gitlab
       Gitlab::Popen.popen(cmd)
     end
 
+    # We're "re-creating" the repo URL because ENV['CI_REPOSITORY_URL'] contains
+    # redacted credentials (e.g. "***:****") which are useless in instructions
+    # the job gives.
+    def ce_public_repo_url
+      "#{ce_project_url}.git"
+    end
+
     def applies_cleanly_msg(branch)
       %Q{
         #{PLEASE_READ_THIS_BANNER}
@@ -288,13 +322,15 @@ module Gitlab
     end
 
     def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
+      ee_repos = ee_remotes.values.uniq
+
       %Q{
         #{PLEASE_READ_THIS_BANNER}
         💥 Oh no! 💥
 
         The `#{ce_branch}` branch does not apply cleanly to the current
         EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch
-        was found in the EE repository.
+        was found in #{ee_repos.join(' nor in ')}.
 
         If you're a community contributor, don't worry, someone from
         GitLab Inc. will take care of this, and you don't have to do anything.
@@ -314,17 +350,17 @@ module Gitlab
         1. Create a new branch from master and cherry-pick your CE commits
 
           # In the EE repo
-          $ git fetch #{EE_REPO_URL} master
+          $ git fetch #{CANONICAL_EE_REPO_URL} master
           $ git checkout -b #{ee_branch_prefix} FETCH_HEAD
-          $ git fetch #{ce_repo_url} #{ce_branch}
+          $ git fetch #{ce_public_repo_url} #{ce_branch}
           $ git cherry-pick SHA # Repeat for all the commits you want to pick
 
-          You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit.
+          Note: You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit.
 
         2. Apply your branch's patch to EE
 
           # In the EE repo
-          $ git fetch #{EE_REPO_URL} master
+          $ git fetch #{CANONICAL_EE_REPO_URL} master
           $ git checkout -b #{ee_branch_prefix} FETCH_HEAD
           $ wget #{patch_url} && git apply --3way #{ce_patch_name}
 
@@ -356,10 +392,9 @@ module Gitlab
         ⚠️ Also, don't forget to create a new merge request on gitlab-ee and
         cross-link it with the CE merge request.
 
-        Once this is done, you can retry this failed build, and it should pass.
+        Once this is done, you can retry this failed job, and it should pass.
 
-        Stay 💪 ! For more information, see
-        https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html
+        #{STAY_STRONG_LINK_TO_DOCS}
         #{THANKS_FOR_READING_BANNER}
       }
     end
@@ -371,16 +406,15 @@ module Gitlab
 
         The `#{ce_branch}` does not apply cleanly to the current EE/master, and
         even though a `#{ee_branch_found}` branch
-        exists in the EE repository, it does not apply cleanly either to
+        exists in #{ee_repo_url}, it does not apply cleanly either to
         EE/master!
 
         #{conflicting_files_msg}
 
         Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and
-        retry this build.
+        retry this job.
 
-        Stay 💪 ! For more information, see
-        https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html
+        #{STAY_STRONG_LINK_TO_DOCS}
         #{THANKS_FOR_READING_BANNER}
       }
     end