Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42

7 files changed, 223 insertions, 134 deletions

diff --git a/spec/support/helpers/cycle_analytics_helpers.rb b/spec/support/helpers/cycle_analytics_helpers.rb
index 722d484609c..70b794f7d82 100644
--- a/spec/support/helpers/cycle_analytics_helpers.rb
+++ b/spec/support/helpers/cycle_analytics_helpers.rb
@@ -59,7 +59,7 @@ module CycleAnalyticsHelpers
   def save_value_stream(custom_value_stream_name)
     fill_in 'create-value-stream-name', with: custom_value_stream_name
 
-    page.find_button(s_('CreateValueStreamForm|Create Value Stream')).click
+    page.find_button(s_('CreateValueStreamForm|Create value stream')).click
     wait_for_requests
   end
 
diff --git a/spec/support/helpers/gitaly_setup.rb b/spec/support/helpers/gitaly_setup.rb
index 923051a2e04..905c439f4d9 100644
--- a/spec/support/helpers/gitaly_setup.rb
+++ b/spec/support/helpers/gitaly_setup.rb
@@ -9,8 +9,13 @@
 require 'securerandom'
 require 'socket'
 require 'logger'
+require 'bundler'
 
 module GitalySetup
+  extend self
+
+  REPOS_STORAGE = 'default'
+
   LOGGER = begin
     default_name = ENV['CI'] ? 'DEBUG' : 'WARN'
     level_name = ENV['GITLAB_TESTING_LOG_LEVEL']&.upcase
@@ -52,11 +57,13 @@ module GitalySetup
 
   def env
     {
-      'HOME' => expand_path('tmp/tests'),
       'GEM_PATH' => Gem.path.join(':'),
-      'BUNDLE_APP_CONFIG' => File.join(gemfile_dir, '.bundle'),
       'BUNDLE_INSTALL_FLAGS' => nil,
+      'BUNDLE_IGNORE_CONFIG' => '1',
+      'BUNDLE_PATH' => bundle_path,
       'BUNDLE_GEMFILE' => gemfile,
+      'BUNDLE_JOBS' => '4',
+      'BUNDLE_RETRY' => '3',
       'RUBYOPT' => nil,
 
       # Git hooks can't run during tests as the internal API is not running.
@@ -65,17 +72,20 @@ module GitalySetup
     }
   end
 
-  # rubocop:disable GitlabSecurity/SystemCommandInjection
-  def set_bundler_config
-    system('bundle config set --local jobs 4', chdir: gemfile_dir)
-    system('bundle config set --local retry 3', chdir: gemfile_dir)
+  def bundle_path
+    # Allow the user to override BUNDLE_PATH if they need to
+    return ENV['GITALY_TEST_BUNDLE_PATH'] if ENV['GITALY_TEST_BUNDLE_PATH']
 
     if ENV['CI']
-      bundle_path = expand_path('vendor/gitaly-ruby')
-      system('bundle', 'config', 'set', '--local', 'path', bundle_path, chdir: gemfile_dir)
+      expand_path('vendor/gitaly-ruby')
+    else
+      explicit_path = Bundler.configured_bundle_path.explicit_path
+
+      return unless explicit_path
+
+      expand_path(explicit_path)
     end
   end
-  # rubocop:enable GitlabSecurity/SystemCommandInjection
 
   def config_path(service)
     case service
@@ -88,6 +98,10 @@ module GitalySetup
     end
   end
 
+  def repos_path(storage = REPOS_STORAGE)
+    Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path
+  end
+
   def service_binary(service)
     case service
     when :gitaly, :gitaly2
@@ -97,16 +111,20 @@ module GitalySetup
     end
   end
 
+  def run_command(cmd, env: {})
+    system(env, *cmd, exception: true, chdir: tmp_tests_gitaly_dir)
+  end
+
   def install_gitaly_gems
-    system(env, "make #{tmp_tests_gitaly_dir}/.ruby-bundle", chdir: tmp_tests_gitaly_dir) # rubocop:disable GitlabSecurity/SystemCommandInjection
+    run_command(%W[make #{tmp_tests_gitaly_dir}/.ruby-bundle], env: env)
   end
 
   def build_gitaly
-    system(env.merge({ 'GIT_VERSION' => nil }), 'make all git', chdir: tmp_tests_gitaly_dir) # rubocop:disable GitlabSecurity/SystemCommandInjection
+    run_command(%w[make all git], env: env.merge('GIT_VERSION' => nil))
   end
 
-  def start_gitaly
-    start(:gitaly)
+  def start_gitaly(toml = nil)
+    start(:gitaly, toml)
   end
 
   def start_gitaly2
@@ -117,14 +135,20 @@ module GitalySetup
     start(:praefect)
   end
 
-  def start(service)
+  def start(service, toml = nil)
+    toml ||= config_path(service)
     args = ["#{tmp_tests_gitaly_bin_dir}/#{service_binary(service)}"]
     args.push("-config") if service == :praefect
-    args.push(config_path(service))
+    args.push(toml)
+
+    # Ensure user configuration does not affect Git
+    # Context: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58776#note_547613780
+    env = self.env.merge('HOME' => nil, 'XDG_CONFIG_HOME' => nil)
+
     pid = spawn(env, *args, [:out, :err] => "log/#{service}-test.log")
 
     begin
-      try_connect!(service)
+      try_connect!(service, toml)
     rescue StandardError
       Process.kill('TERM', pid)
       raise
@@ -161,29 +185,37 @@ module GitalySetup
     abort 'bundle check failed' unless system(env, 'bundle', 'check', out: out, chdir: gemfile_dir)
   end
 
-  def read_socket_path(service)
+  def connect_proc(toml)
     # This code needs to work in an environment where we cannot use bundler,
     # so we cannot easily use the toml-rb gem. This ad-hoc parser should be
     # good enough.
-    config_text = IO.read(config_path(service))
+    config_text = IO.read(toml)
 
     config_text.lines.each do |line|
-      match_data = line.match(/^\s*socket_path\s*=\s*"([^"]*)"$/)
+      match_data = line.match(/^\s*(socket_path|listen_addr)\s*=\s*"([^"]*)"$/)
 
-      return match_data[1] if match_data
+      next unless match_data
+
+      case match_data[1]
+      when 'socket_path'
+        return -> { UNIXSocket.new(match_data[2]) }
+      when 'listen_addr'
+        addr, port = match_data[2].split(':')
+        return -> { TCPSocket.new(addr, port.to_i) }
+      end
     end
 
-    raise "failed to find socket_path in #{config_path(service)}"
+    raise "failed to find socket_path or listen_addr in #{toml}"
   end
 
-  def try_connect!(service)
+  def try_connect!(service, toml)
     LOGGER.debug "Trying to connect to #{service}: "
     timeout = 20
     delay = 0.1
-    socket = read_socket_path(service)
+    connect = connect_proc(toml)
 
     Integer(timeout / delay).times do
-      UNIXSocket.new(socket)
+      connect.call
       LOGGER.debug " OK\n"
 
       return
@@ -194,6 +226,128 @@ module GitalySetup
 
     LOGGER.warn " FAILED to connect to #{service}\n"
 
-    raise "could not connect to #{socket}"
+    raise "could not connect to #{service}"
+  end
+
+  def gitaly_socket_path
+    Gitlab::GitalyClient.address(REPOS_STORAGE).delete_prefix('unix:')
+  end
+
+  def gitaly_dir
+    socket_path = gitaly_socket_path
+    socket_path = File.expand_path(gitaly_socket_path) if expand_path_for_socket?
+
+    File.dirname(socket_path)
+  end
+
+  # Linux fails with "bind: invalid argument" if a UNIX socket path exceeds 108 characters:
+  # https://github.com/golang/go/issues/6895. We use absolute paths in CI to ensure
+  # that changes in the current working directory don't affect GRPC reconnections.
+  def expand_path_for_socket?
+    !!ENV['CI']
+  end
+
+  def setup_gitaly
+    unless ENV['CI']
+      # In CI Gitaly is built in the setup-test-env job and saved in the
+      # artifacts. So when tests are started, there's no need to build Gitaly.
+      build_gitaly
+    end
+
+    Gitlab::SetupHelper::Gitaly.create_configuration(
+      gitaly_dir,
+      { 'default' => repos_path },
+      force: true,
+      options: {
+        prometheus_listen_addr: 'localhost:9236'
+      }
+    )
+    Gitlab::SetupHelper::Gitaly.create_configuration(
+      gitaly_dir,
+      { 'default' => repos_path },
+      force: true,
+      options: {
+        internal_socket_dir: File.join(gitaly_dir, "internal_gitaly2"),
+        gitaly_socket: "gitaly2.socket",
+        config_filename: "gitaly2.config.toml"
+      }
+    )
+    Gitlab::SetupHelper::Praefect.create_configuration(gitaly_dir, { 'praefect' => repos_path }, force: true)
+  end
+
+  def socket_path(service)
+    File.join(tmp_tests_gitaly_dir, "#{service}.socket")
+  end
+
+  def praefect_socket_path
+    "unix:" + socket_path(:praefect)
+  end
+
+  def stop(pid)
+    Process.kill('KILL', pid)
+  rescue Errno::ESRCH
+    # The process can already be gone if the test run was INTerrupted.
+  end
+
+  def spawn_gitaly(toml = nil)
+    check_gitaly_config!
+
+    pids = []
+
+    if toml
+      pids << start_gitaly(toml)
+    else
+      pids << start_gitaly
+      pids << start_gitaly2
+      pids << start_praefect
+    end
+
+    Kernel.at_exit do
+      # In CI, this function is called by scripts/gitaly-test-spawn, triggered
+      # in a before_script. Gitaly needs to remain running until the container
+      # is stopped.
+      next if ENV['CI']
+      # In Workhorse tests (locally or in CI), this function is called by
+      # scripts/gitaly-test-spawn during `make test`. Gitaly needs to remain
+      # running until `make test` cleans it up.
+      next if ENV['GITALY_PID_FILE']
+
+      pids.each { |pid| stop(pid) }
+    end
+  rescue StandardError
+    raise gitaly_failure_message
+  end
+
+  def gitaly_failure_message
+    message = "gitaly spawn failed\n\n"
+
+    message += "- The `gitaly` binary does not exist: #{gitaly_binary}\n" unless File.exist?(gitaly_binary)
+    message += "- The `praefect` binary does not exist: #{praefect_binary}\n" unless File.exist?(praefect_binary)
+    message += "- The `git` binary does not exist: #{git_binary}\n" unless File.exist?(git_binary)
+
+    message += "\nCheck log/gitaly-test.log for errors.\n"
+
+    unless ci?
+      message += "\nIf binaries are missing, try running `make -C tmp/tests/gitaly build git.`\n"
+      message += "\nOtherwise, try running `rm -rf #{tmp_tests_gitaly_dir}`."
+    end
+
+    message
+  end
+
+  def git_binary
+    File.join(tmp_tests_gitaly_dir, "_build", "deps", "git", "install", "bin", "git")
+  end
+
+  def gitaly_binary
+    File.join(tmp_tests_gitaly_dir, "_build", "bin", "gitaly")
+  end
+
+  def praefect_binary
+    File.join(tmp_tests_gitaly_dir, "_build", "bin", "praefect")
+  end
+
+  def git_binary_exists?
+    File.exist?(git_binary)
   end
 end
diff --git a/spec/support/helpers/login_helpers.rb b/spec/support/helpers/login_helpers.rb
index d9157fa7485..4e0e8dd96ee 100644
--- a/spec/support/helpers/login_helpers.rb
+++ b/spec/support/helpers/login_helpers.rb
@@ -95,7 +95,7 @@ module LoginHelpers
     visit new_user_session_path
 
     fill_in "user_login", with: user.email
-    fill_in "user_password", with: "12345678"
+    fill_in "user_password", with: Gitlab::Password.test_default
     check 'user_remember_me' if remember
 
     click_button "Sign in"
diff --git a/spec/support/helpers/stub_gitlab_calls.rb b/spec/support/helpers/stub_gitlab_calls.rb
index ae031f58bd4..c3459f7bc81 100644
--- a/spec/support/helpers/stub_gitlab_calls.rb
+++ b/spec/support/helpers/stub_gitlab_calls.rb
@@ -92,12 +92,7 @@ module StubGitlabCalls
   end
 
   def stub_commonmark_sourcepos_disabled
-    render_options =
-      if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-        Banzai::Filter::MarkdownEngines::CommonMark::RENDER_OPTIONS_C
-      else
-        Banzai::Filter::MarkdownEngines::CommonMark::RENDER_OPTIONS_RUBY
-      end
+    render_options = Banzai::Filter::MarkdownEngines::CommonMark::RENDER_OPTIONS
 
     allow_any_instance_of(Banzai::Filter::MarkdownEngines::CommonMark)
       .to receive(:render_options)
diff --git a/spec/support/helpers/stub_object_storage.rb b/spec/support/helpers/stub_object_storage.rb
index 5e86b08aa45..d49a14f7f5b 100644
--- a/spec/support/helpers/stub_object_storage.rb
+++ b/spec/support/helpers/stub_object_storage.rb
@@ -91,6 +91,12 @@ module StubObjectStorage
                                  **params)
   end
 
+  def stub_ci_secure_file_object_storage(**params)
+    stub_object_storage_uploader(config: Gitlab.config.ci_secure_files.object_store,
+                                 uploader: Ci::SecureFileUploader,
+                                 **params)
+  end
+
   def stub_terraform_state_object_storage(**params)
     stub_object_storage_uploader(config: Gitlab.config.terraform_state.object_store,
                                  uploader: Terraform::StateUploader,
diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb
index d36bc4e3cb4..5c3ca92c4d0 100644
--- a/spec/support/helpers/test_env.rb
+++ b/spec/support/helpers/test_env.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'parallel'
+require_relative 'gitaly_setup'
 
 module TestEnv
   extend self
@@ -93,7 +94,6 @@ module TestEnv
   }.freeze
 
   TMP_TEST_PATH = Rails.root.join('tmp', 'tests').freeze
-  REPOS_STORAGE = 'default'
   SECOND_STORAGE_PATH = Rails.root.join('tmp', 'tests', 'second_storage')
   SETUP_METHODS = %i[setup_gitaly setup_gitlab_shell setup_workhorse setup_factory_repo setup_forked_repo].freeze
 
@@ -128,7 +128,7 @@ module TestEnv
 
   # Can be overriden
   def post_init
-    start_gitaly(gitaly_dir)
+    start_gitaly
   end
 
   # Clean /tmp/tests
@@ -142,12 +142,15 @@ module TestEnv
     end
 
     FileUtils.mkdir_p(
-      Gitlab::GitalyClient::StorageSettings.allow_disk_access { TestEnv.repos_path }
+      Gitlab::GitalyClient::StorageSettings.allow_disk_access { GitalySetup.repos_path }
     )
     FileUtils.mkdir_p(SECOND_STORAGE_PATH)
     FileUtils.mkdir_p(backup_path)
     FileUtils.mkdir_p(pages_path)
     FileUtils.mkdir_p(artifacts_path)
+    FileUtils.mkdir_p(lfs_path)
+    FileUtils.mkdir_p(terraform_state_path)
+    FileUtils.mkdir_p(packages_path)
   end
 
   def setup_gitlab_shell
@@ -156,111 +159,28 @@ module TestEnv
 
   def setup_gitaly
     component_timed_setup('Gitaly',
-      install_dir: gitaly_dir,
+      install_dir: GitalySetup.gitaly_dir,
       version: Gitlab::GitalyClient.expected_server_version,
-      task: "gitlab:gitaly:test_install",
-      task_args: [gitaly_dir, repos_path, gitaly_url].compact) do
-        Gitlab::SetupHelper::Gitaly.create_configuration(
-          gitaly_dir,
-          { 'default' => repos_path },
-          force: true,
-          options: {
-            prometheus_listen_addr: 'localhost:9236'
-          }
-        )
-        Gitlab::SetupHelper::Gitaly.create_configuration(
-          gitaly_dir,
-          { 'default' => repos_path },
-          force: true,
-          options: {
-            internal_socket_dir: File.join(gitaly_dir, "internal_gitaly2"),
-            gitaly_socket: "gitaly2.socket",
-            config_filename: "gitaly2.config.toml"
-          }
-        )
-        Gitlab::SetupHelper::Praefect.create_configuration(gitaly_dir, { 'praefect' => repos_path }, force: true)
-      end
-  end
-
-  def gitaly_socket_path
-    Gitlab::GitalyClient.address('default').sub(/\Aunix:/, '')
-  end
-
-  def gitaly_dir
-    socket_path = gitaly_socket_path
-    socket_path = File.expand_path(gitaly_socket_path) if expand_path?
-
-    File.dirname(socket_path)
-  end
-
-  # Linux fails with "bind: invalid argument" if a UNIX socket path exceeds 108 characters:
-  # https://github.com/golang/go/issues/6895. We use absolute paths in CI to ensure
-  # that changes in the current working directory don't affect GRPC reconnections.
-  def expand_path?
-    !!ENV['CI']
+      task: "gitlab:gitaly:clone",
+      fresh_install: ENV.key?('FORCE_GITALY_INSTALL'),
+      task_args: [GitalySetup.gitaly_dir, GitalySetup.repos_path, gitaly_url].compact) do
+      GitalySetup.setup_gitaly
+    end
   end
 
-  def start_gitaly(gitaly_dir)
+  def start_gitaly
     if ci?
       # Gitaly has been spawned outside this process already
       return
     end
 
-    spawn_script = Rails.root.join('scripts/gitaly-test-spawn').to_s
-    Bundler.with_original_env do
-      unless system(spawn_script)
-        message = 'gitaly spawn failed'
-        message += " (try `rm -rf #{gitaly_dir}` ?)" unless ci?
-        raise message
-      end
-    end
-
-    gitaly_pid = Integer(File.read(TMP_TEST_PATH.join('gitaly.pid')))
-    gitaly2_pid = Integer(File.read(TMP_TEST_PATH.join('gitaly2.pid')))
-    praefect_pid = Integer(File.read(TMP_TEST_PATH.join('praefect.pid')))
-
-    Kernel.at_exit do
-      pids = [gitaly_pid, gitaly2_pid, praefect_pid]
-      pids.each { |pid| stop(pid) }
-    end
-
-    wait('gitaly')
-    wait('praefect')
-  end
-
-  def stop(pid)
-    Process.kill('KILL', pid)
-  rescue Errno::ESRCH
-    # The process can already be gone if the test run was INTerrupted.
+    GitalySetup.spawn_gitaly
   end
 
   def gitaly_url
     ENV.fetch('GITALY_REPO_URL', nil)
   end
 
-  def socket_path(service)
-    TMP_TEST_PATH.join('gitaly', "#{service}.socket").to_s
-  end
-
-  def praefect_socket_path
-    "unix:" + socket_path(:praefect)
-  end
-
-  def wait(service)
-    sleep_time = 10
-    sleep_interval = 0.1
-    socket = socket_path(service)
-
-    Integer(sleep_time / sleep_interval).times do
-      Socket.unix(socket)
-      return
-    rescue StandardError
-      sleep sleep_interval
-    end
-
-    raise "could not connect to #{service} at #{socket.inspect} after #{sleep_time} seconds"
-  end
-
   # Feature specs are run through Workhorse
   def setup_workhorse
     # Always rebuild the config file
@@ -376,8 +296,7 @@ module TestEnv
 
   def rm_storage_dir(storage, dir)
     Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-      repos_path = Gitlab.config.repositories.storages[storage].legacy_disk_path
-      target_repo_refs_path = File.join(repos_path, dir)
+      target_repo_refs_path = File.join(GitalySetup.repos_path(storage), dir)
       FileUtils.remove_dir(target_repo_refs_path)
     end
   rescue Errno::ENOENT
@@ -385,8 +304,7 @@ module TestEnv
 
   def storage_dir_exists?(storage, dir)
     Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-      repos_path = Gitlab.config.repositories.storages[storage].legacy_disk_path
-      File.exist?(File.join(repos_path, dir))
+      File.exist?(File.join(GitalySetup.repos_path(storage), dir))
     end
   end
 
@@ -399,7 +317,7 @@ module TestEnv
   end
 
   def repos_path
-    @repos_path ||= Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path
+    @repos_path ||= GitalySetup.repos_path
   end
 
   def backup_path
@@ -414,6 +332,18 @@ module TestEnv
     Gitlab.config.artifacts.storage_path
   end
 
+  def lfs_path
+    Gitlab.config.lfs.storage_path
+  end
+
+  def terraform_state_path
+    Gitlab.config.terraform_state.storage_path
+  end
+
+  def packages_path
+    Gitlab.config.packages.storage_path
+  end
+
   # When no cached assets exist, manually hit the root path to create them
   #
   # Otherwise they'd be created by the first test, often timing out and
@@ -512,7 +442,7 @@ module TestEnv
     end
   end
 
-  def component_timed_setup(component, install_dir:, version:, task:, task_args: [])
+  def component_timed_setup(component, install_dir:, version:, task:, fresh_install: true, task_args: [])
     start = Time.now
 
     ensure_component_dir_name_is_correct!(component, install_dir)
@@ -522,7 +452,7 @@ module TestEnv
 
     if component_needs_update?(install_dir, version)
       # Cleanup the component entirely to ensure we start fresh
-      FileUtils.rm_rf(install_dir)
+      FileUtils.rm_rf(install_dir) if fresh_install
 
       if ENV['SKIP_RAILS_ENV_IN_RAKE']
         # When we run `scripts/setup-test-env`, we take care of loading the necessary dependencies
diff --git a/spec/support/helpers/usage_data_helpers.rb b/spec/support/helpers/usage_data_helpers.rb
index 5865bafd382..776ea37ffdc 100644
--- a/spec/support/helpers/usage_data_helpers.rb
+++ b/spec/support/helpers/usage_data_helpers.rb
@@ -183,6 +183,10 @@ module UsageDataHelpers
       )
   end
 
+  def stub_database_flavor_check(flavor = nil)
+    allow(ApplicationRecord.database).to receive(:flavor).and_return(flavor)
+  end
+
   def clear_memoized_values(values)
     values.each { |v| described_class.clear_memoization(v) }
   end