From 0241c182179aa793e068da4ed6ac44963f72514e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Mon, 5 Mar 2018 17:24:26 +0100 Subject: Improve the ee_compat_check_job to handle gitlab-ee forks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/ee_compat_check.rb | 148 ++++++++++++++++++++++++++---------------- 1 file changed, 91 insertions(+), 57 deletions(-) diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 0fb71976883..5fdd5dcd374 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -2,8 +2,8 @@ module Gitlab # Checks if a set of migrations requires downtime or not. class EeCompatCheck - DEFAULT_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze - EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze + CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze + CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb}i.freeze PLEASE_READ_THIS_BANNER = %Q{ @@ -11,57 +11,81 @@ module Gitlab ===================== PLEASE READ THIS ===================== ============================================================ }.freeze + STAY_STRONG_LINK_TO_DOCS = %Q{ + Stay 💪! For more information, see + https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html + }.freeze THANKS_FOR_READING_BANNER = %Q{ ============================================================ ==================== THANKS FOR READING ==================== ============================================================\n }.freeze - attr_reader :ee_repo_dir, :patches_dir, :ce_project_url, :ce_repo_url, :ce_branch, :ee_branch_found + attr_reader :ee_repo_dir, :patches_dir + attr_reader :ce_project_url, :ee_repo_url + attr_reader :ce_branch, :ee_remote_with_branch, :ee_branch_found attr_reader :job_id, :failed_files - def initialize(branch:, ce_project_url: DEFAULT_CE_PROJECT_URL, job_id: nil) + def initialize(branch:, ce_project_url: CANONICAL_CE_PROJECT_URL, job_id: nil) @ee_repo_dir = CHECK_DIR.join('ee-repo') @patches_dir = CHECK_DIR.join('patches') @ce_branch = branch @ce_project_url = ce_project_url - @ce_repo_url = "#{ce_project_url}.git" + @ee_repo_url = ce_public_repo_url.sub('gitlab-ce', 'gitlab-ee') @job_id = job_id end def check ensure_patches_dir - add_remote('canonical-ce', "#{DEFAULT_CE_PROJECT_URL}.git") - generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, remote: 'canonical-ce') + # We're generating the patch against the canonical-ce remote since forks' + # master branch are not necessarily up-to-date. + add_remote('canonical-ce', "#{CANONICAL_CE_PROJECT_URL}.git") + generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, branch_remote: 'origin', master_remote: 'canonical-ce') ensure_ee_repo Dir.chdir(ee_repo_dir) do step("In the #{ee_repo_dir} directory") - add_remote('canonical-ee', EE_REPO_URL) + ee_remotes.each do |key, url| + add_remote(key, url) + end + fetch(branch: 'master', depth: 20, remote: 'canonical-ee') status = catch(:halt_check) do ce_branch_compat_check! delete_ee_branches_locally! ee_branch_presence_check! - step("Checking out #{ee_branch_found}", %W[git checkout -b #{ee_branch_found} canonical-ee/#{ee_branch_found}]) - generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, remote: 'canonical-ee') + step("Checking out #{ee_remote_with_branch}/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} #{ee_remote_with_branch}/#{ee_branch_found}]) + generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, branch_remote: ee_remote_with_branch, master_remote: 'canonical-ee') ee_branch_compat_check! end delete_ee_branches_locally! - if status.nil? - true - else - false - end + status.nil? end end private + def fork? + ce_project_url != CANONICAL_CE_PROJECT_URL + end + + def ee_remotes + return @ee_remotes if defined?(@ee_remotes) + + remotes = + { + 'ee' => ee_repo_url, + 'canonical-ee' => CANONICAL_EE_REPO_URL + } + remotes.delete('ee') unless fork? + + @ee_remotes = remotes + end + def add_remote(name, url) step( "Adding the #{name} remote (#{url})", @@ -70,28 +94,32 @@ module Gitlab end def ensure_ee_repo - if Dir.exist?(ee_repo_dir) - step("#{ee_repo_dir} already exists") - else - step( - "Cloning #{EE_REPO_URL} into #{ee_repo_dir}", - %W[git clone --branch master --single-branch --depth=200 #{EE_REPO_URL} #{ee_repo_dir}] - ) + unless clone_repo(ee_repo_url, ee_repo_dir) + # Fallback to using the canonical EE if there is no forked EE + clone_repo(CANONICAL_EE_REPO_URL, ee_repo_dir) end end + def clone_repo(url, dir) + _, status = step( + "Cloning #{url} into #{dir}", + %W[git clone --branch master --single-branch --depth=200 #{url} #{dir}] + ) + status.zero? + end + def ensure_patches_dir FileUtils.mkdir_p(patches_dir) end - def generate_patch(branch:, patch_path:, remote:) + def generate_patch(branch:, patch_path:, branch_remote:, master_remote:) FileUtils.rm(patch_path, force: true) - find_merge_base_with_master(branch: branch, master_remote: remote) + find_merge_base_with_master(branch: branch, branch_remote: branch_remote, master_remote: master_remote) step( - "Generating the patch against #{remote}/master in #{patch_path}", - %W[git diff --binary #{remote}/master...origin/#{branch}] + "Generating the patch against #{master_remote}/master in #{patch_path}", + %W[git diff --binary #{master_remote}/master...#{branch_remote}/#{branch}] ) do |output, status| throw(:halt_check, :ko) unless status.zero? @@ -109,23 +137,22 @@ module Gitlab end def ee_branch_presence_check! - _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch canonical-ee #{ee_branch_prefix}]) - - if status.zero? - @ee_branch_found = ee_branch_prefix - return + ee_remotes.keys.each do |remote| + [ee_branch_prefix, ee_branch_suffix].each do |branch| + _, status = step("Fetching #{remote}/#{ee_branch_prefix}", %W[git fetch #{remote} #{branch}]) + + if status.zero? + @ee_remote_with_branch = remote + @ee_branch_found = branch + return true + end + end end - _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch canonical-ee #{ee_branch_suffix}]) - - if status.zero? - @ee_branch_found = ee_branch_suffix - else - puts - puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + puts + puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg - throw(:halt_check, :ko) - end + throw(:halt_check, :ko) end def ee_branch_compat_check! @@ -181,10 +208,10 @@ module Gitlab command(%W[git branch --delete --force #{ee_branch_suffix}]) end - def merge_base_found?(master_remote:, branch:) + def merge_base_found?(branch:, branch_remote:, master_remote:) step( "Finding merge base with #{master_remote}/master", - %W[git merge-base #{master_remote}/master origin/#{branch}] + %W[git merge-base #{master_remote}/master #{branch_remote}/#{branch}] ) do |output, status| if status.zero? puts "Merge base was found: #{output}" @@ -193,7 +220,7 @@ module Gitlab end end - def find_merge_base_with_master(branch:, master_remote:) + def find_merge_base_with_master(branch:, branch_remote:, master_remote:) # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403) # In total we go (20 + 54 + 148 + 403 = 625) commits deeper depth = 20 @@ -202,10 +229,10 @@ module Gitlab depth += Math.exp(factor).to_i # Repository is initially cloned with a depth of 20 so we need to fetch # deeper in the case the branch has more than 20 commits on top of master - fetch(branch: branch, depth: depth, remote: 'origin') + fetch(branch: branch, depth: depth, remote: branch_remote) fetch(branch: 'master', depth: depth, remote: master_remote) - merge_base_found?(master_remote: master_remote, branch: branch) + merge_base_found?(branch: branch, branch_remote: branch_remote, master_remote: master_remote) end raise "\n#{branch} is too far behind #{master_remote}/master, please rebase it!\n" unless success @@ -274,6 +301,13 @@ module Gitlab Gitlab::Popen.popen(cmd) end + # We're "re-creating" the repo URL because ENV['CI_REPOSITORY_URL'] contains + # redacted credentials (e.g. "***:****") which are useless in instructions + # the job gives. + def ce_public_repo_url + "#{ce_project_url}.git" + end + def applies_cleanly_msg(branch) %Q{ #{PLEASE_READ_THIS_BANNER} @@ -288,13 +322,15 @@ module Gitlab end def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + ee_repos = ee_remotes.values.uniq + %Q{ #{PLEASE_READ_THIS_BANNER} 💥 Oh no! 💥 The `#{ce_branch}` branch does not apply cleanly to the current EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch - was found in the EE repository. + was found in #{ee_repos.join(' nor in ')}. If you're a community contributor, don't worry, someone from GitLab Inc. will take care of this, and you don't have to do anything. @@ -314,17 +350,17 @@ module Gitlab 1. Create a new branch from master and cherry-pick your CE commits # In the EE repo - $ git fetch #{EE_REPO_URL} master + $ git fetch #{CANONICAL_EE_REPO_URL} master $ git checkout -b #{ee_branch_prefix} FETCH_HEAD - $ git fetch #{ce_repo_url} #{ce_branch} + $ git fetch #{ce_public_repo_url} #{ce_branch} $ git cherry-pick SHA # Repeat for all the commits you want to pick - You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit. + Note: You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit. 2. Apply your branch's patch to EE # In the EE repo - $ git fetch #{EE_REPO_URL} master + $ git fetch #{CANONICAL_EE_REPO_URL} master $ git checkout -b #{ee_branch_prefix} FETCH_HEAD $ wget #{patch_url} && git apply --3way #{ce_patch_name} @@ -356,10 +392,9 @@ module Gitlab ⚠️ Also, don't forget to create a new merge request on gitlab-ee and cross-link it with the CE merge request. - Once this is done, you can retry this failed build, and it should pass. + Once this is done, you can retry this failed job, and it should pass. - Stay 💪 ! For more information, see - https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html + #{STAY_STRONG_LINK_TO_DOCS} #{THANKS_FOR_READING_BANNER} } end @@ -371,16 +406,15 @@ module Gitlab The `#{ce_branch}` does not apply cleanly to the current EE/master, and even though a `#{ee_branch_found}` branch - exists in the EE repository, it does not apply cleanly either to + exists in #{ee_repo_url}, it does not apply cleanly either to EE/master! #{conflicting_files_msg} Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and - retry this build. + retry this job. - Stay 💪 ! For more information, see - https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html + #{STAY_STRONG_LINK_TO_DOCS} #{THANKS_FOR_READING_BANNER} } end -- cgit v1.2.1