Add latest changes from gitlab-org/gitlab@15-5-stable-eev15.5.0-rc42

13 files changed, 93 insertions, 136 deletions

diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 3fc4b56f458..45307c3454b 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -183,6 +183,7 @@ Settings.gitlab['default_project_creation'] ||= ::Gitlab::Access::DEVELOPER_MAIN
 Settings.gitlab['default_project_deletion_protection'] ||= false
 Settings.gitlab['default_projects_limit'] ||= 100000
 Settings.gitlab['default_branch_protection'] ||= 2
+# `default_can_create_group` is deprecated since GitLab 15.5 in favour of the `can_create_group` column on `ApplicationSetting`.
 Settings.gitlab['default_can_create_group'] = true if Settings.gitlab['default_can_create_group'].nil?
 Settings.gitlab['default_theme'] = Gitlab::Themes::APPLICATION_DEFAULT if Settings.gitlab['default_theme'].nil?
 Settings.gitlab['host'] ||= ENV['GITLAB_HOST'] || 'localhost'
@@ -456,6 +457,7 @@ if Gitlab.ee? && Settings['ee_cron_jobs']
   Settings.cron_jobs.merge!(Settings.ee_cron_jobs)
 end
 
+Settings.cron_jobs['poll_interval'] ||= 30
 Settings.cron_jobs['stuck_ci_jobs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_ci_jobs_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['stuck_ci_jobs_worker']['job_class'] = 'StuckCiJobsWorker'
@@ -655,7 +657,7 @@ Settings.cron_jobs['ci_runner_versions_reconciliation_worker'] ||= Settingslogic
 Settings.cron_jobs['ci_runner_versions_reconciliation_worker']['cron'] ||= '@daily'
 Settings.cron_jobs['ci_runner_versions_reconciliation_worker']['job_class'] = 'Ci::Runners::ReconcileExistingRunnerVersionsCronWorker'
 Settings.cron_jobs['users_migrate_records_to_ghost_user_in_batches_worker'] ||= Settingslogic.new({})
-Settings.cron_jobs['users_migrate_records_to_ghost_user_in_batches_worker']['cron'] ||= '*/1 * * * *'
+Settings.cron_jobs['users_migrate_records_to_ghost_user_in_batches_worker']['cron'] ||= '*/2 * * * *'
 Settings.cron_jobs['users_migrate_records_to_ghost_user_in_batches_worker']['job_class'] = 'Users::MigrateRecordsToGhostUserInBatchesWorker'
 
 Gitlab.ee do
@@ -779,9 +781,9 @@ Gitlab.ee do
   Settings.cron_jobs['security_orchestration_policy_rule_schedule_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['security_orchestration_policy_rule_schedule_worker']['cron'] ||= '*/15 * * * *'
   Settings.cron_jobs['security_orchestration_policy_rule_schedule_worker']['job_class'] = 'Security::OrchestrationPolicyRuleScheduleWorker'
-  Settings.cron_jobs['security_findings_cleanup_worker'] ||= Settingslogic.new({})
-  Settings.cron_jobs['security_findings_cleanup_worker']['cron'] ||= '0 */4 * * 6,0'
-  Settings.cron_jobs['security_findings_cleanup_worker']['job_class'] = 'Security::Findings::CleanupWorker'
+  Settings.cron_jobs['security_scans_purge_worker'] ||= Settingslogic.new({})
+  Settings.cron_jobs['security_scans_purge_worker']['cron'] ||= '0 */4 * * 6,0'
+  Settings.cron_jobs['security_scans_purge_worker']['job_class'] = 'Security::Scans::PurgeWorker'
   Settings.cron_jobs['app_sec_dast_profile_schedule_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['app_sec_dast_profile_schedule_worker']['cron'] ||= '7-59/15 * * * *'
   Settings.cron_jobs['app_sec_dast_profile_schedule_worker']['job_class'] = 'AppSec::Dast::ProfileScheduleWorker'
@@ -1045,6 +1047,7 @@ Settings.shutdown['blackout_seconds'] ||= 10
 #
 if Rails.env.test?
   Settings.gitlab['default_projects_limit']   = 42
+  # `default_can_create_group` is deprecated since GitLab 15.5 in favour of the `can_create_group` column on `ApplicationSetting`.
   Settings.gitlab['default_can_create_group'] = true
   Settings.gitlab['default_can_create_team']  = false
 end
diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb
index 1e2786db413..7773740fec4 100644
--- a/config/initializers/7_redis.rb
+++ b/config/initializers/7_redis.rb
@@ -4,12 +4,6 @@ require 'gitlab/redis'
 
 Redis.raise_deprecations = true unless Rails.env.production?
 
-# We set the instance variable directly to suppress warnings.
-# We cannot switch to the new behavior until we change all existing `redis.exists` calls to `redis.exists?`.
-# Some gems also need to be updated
-# https://gitlab.com/gitlab-org/gitlab/-/issues/340602
-Redis.instance_variable_set(:@exists_returns_integer, false)
-
 Redis::Client.prepend(Gitlab::Instrumentation::RedisInterceptor)
 
 # Make sure we initialize a Redis connection pool before multi-threaded
diff --git a/config/initializers/attr_encrypted_no_db_connection.rb b/config/initializers/attr_encrypted_no_db_connection.rb
deleted file mode 100644
index d9e943bd249..00000000000
--- a/config/initializers/attr_encrypted_no_db_connection.rb
+++ /dev/null
@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-raise 'This patch is only tested with attr_encrypted v3.1.0' unless AttrEncrypted::Version.string == '3.1.0'
-
-module AttrEncrypted
-  module Adapters
-    module ActiveRecord
-      module GitlabMonkeyPatches
-        # Prevent attr_encrypted from defining virtual accessors for encryption
-        # data when the code and schema are out of sync. See this issue for more
-        # details: https://github.com/attr-encrypted/attr_encrypted/issues/332
-        def attribute_instance_methods_as_symbols_available?
-          false
-        end
-
-        protected
-
-        # The attr_encrypted gem is not actively maintained
-        # At the same time it contains the code that raises kwargs deprecation warnings:
-        # https://github.com/attr-encrypted/attr_encrypted/blob/master/lib/attr_encrypted/adapters/active_record.rb#L65
-        #
-        def attr_encrypted(*attrs)
-          super
-
-          attr = attrs.first
-
-          redefine_method(:"#{attr}_changed?") do |**options|
-            attribute_changed?(attr, **options)
-          end
-        end
-      end
-    end
-  end
-end
-
-# As of v3.1.0, the attr_encrypted gem defines the AttrEncrypted and
-# AttrEncrypted::Adapters::ActiveRecord modules, and uses "extend" to mix them
-# into the ActiveRecord::Base class. This intervention overrides utility methods
-# defined by attr_encrypted to fix two bugs, as detailed above.
-#
-# The methods are used here: https://github.com/attr-encrypted/attr_encrypted/blob/3.1.0/lib/attr_encrypted.rb#L145-158
-ActiveSupport.on_load(:active_record) do
-  extend AttrEncrypted::Adapters::ActiveRecord::GitlabMonkeyPatches
-end
diff --git a/config/initializers/attr_encrypted_thread_safe.rb b/config/initializers/attr_encrypted_thread_safe.rb
deleted file mode 100644
index be0bb56ffdc..00000000000
--- a/config/initializers/attr_encrypted_thread_safe.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-# As of v3.1.0, attr_encrypted is not thread-safe because all instances share the same `encrypted_attributes`
-# This was fixed in https://github.com/attr-encrypted/attr_encrypted/commit/d4ca0e2073ca6ba5035997ce25f7fc0b4bfbe39e
-# but no release was made after that so we have to patch it ourselves here
-
-module AttrEncrypted
-  module InstanceMethods
-    def encrypted_attributes
-      @encrypted_attributes ||= begin
-        duplicated = {}
-        self.class.encrypted_attributes.map { |key, value| duplicated[key] = value.dup }
-        duplicated
-      end
-    end
-  end
-end
diff --git a/config/initializers/countries.rb b/config/initializers/countries.rb
index 9f8592ba641..52537b5d885 100644
--- a/config/initializers/countries.rb
+++ b/config/initializers/countries.rb
@@ -4,8 +4,9 @@ ISO3166.configure do |config|
   config.locales = [:en]
 end
 
-# GitLab permits users to sign up in Ukraine except the Crimean Region: https://about.gitlab.com/handbook/people-operations/code-of-conduct/#trade-compliance-exportimport-control
-# This overrides the display name for Ukraine to Ukraine (except Crimean Region)
+# GitLab permits users to sign up in Ukraine except for the regions: Crimea, Donetsk, and Luhansk: https://about.gitlab.com/handbook/people-operations/code-of-conduct/#trade-compliance-exportimport-control
+# This overrides the display name for Ukraine to 'Ukraine (except the Crimea, Donetsk, and Luhansk regions)'
+# See: https://gitlab.com/gitlab-org/gitlab/-/issues/374946
 # To be removed after https://gitlab.com/gitlab-org/gitlab/issues/14784 is implemented
 # Data fetched is based on https://github.com/hexorx/countries/blob/master/lib/countries/data/countries/UA.yaml
 ISO3166::Data.register(
@@ -22,7 +23,7 @@ ISO3166::Data.register(
   international_prefix: '810',
   ioc: "UKR",
   gec: "UP",
-  name: "Ukraine (except Crimean Region)",
+  name: "Ukraine (except the Crimea, Donetsk, and Luhansk regions)",
   national_destination_code_lengths: [2],
   national_number_lengths: [8, 9],
   national_prefix: '8',
diff --git a/config/initializers/diagnostic_reports.rb b/config/initializers/diagnostic_reports.rb
index b9932822a0b..47266f99f2d 100644
--- a/config/initializers/diagnostic_reports.rb
+++ b/config/initializers/diagnostic_reports.rb
@@ -2,9 +2,7 @@
 
 return unless Gitlab::Utils.to_boolean(ENV['GITLAB_DIAGNOSTIC_REPORTS_ENABLED'])
 
-# Any actions beyond this check should only execute outside of tests,
-# when running in application context (i.e. not in the Rails console or rspec)
-return unless Gitlab::Runtime.application?
+return unless Gitlab::Runtime.puma?
 
 Gitlab::Cluster::LifecycleEvents.on_worker_start do
   Gitlab::Memory::ReportsDaemon.instance.start
diff --git a/config/initializers/licensee_license_patch.rb b/config/initializers/licensee_license_patch.rb
new file mode 100644
index 00000000000..d4680db5071
--- /dev/null
+++ b/config/initializers/licensee_license_patch.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'licensee/license'
+
+module Licensee
+  module LicensePatch
+    # Patch from https://github.com/licensee/licensee/pull/589
+    def ==(other)
+      other.is_a?(self.class) && key == other.key
+    end
+  end
+
+  License.prepend LicensePatch
+end
diff --git a/config/initializers/load_balancing.rb b/config/initializers/load_balancing.rb
index bcc62a6be5d..aa621602573 100644
--- a/config/initializers/load_balancing.rb
+++ b/config/initializers/load_balancing.rb
@@ -2,6 +2,15 @@
 
 Gitlab::Application.configure do |config|
   config.middleware.use(Gitlab::Database::LoadBalancing::RackMiddleware)
+
+  # We need re-rerun the setup when code reloads in development
+  config.reloader.to_prepare do
+    if Gitlab.dev_or_test_env?
+      Gitlab::Database::LoadBalancing.base_models.each do |model|
+        Gitlab::Database::LoadBalancing::Setup.new(model).setup
+      end
+    end
+  end
 end
 
 Gitlab::Database::LoadBalancing.base_models.each do |model|
@@ -14,13 +23,6 @@ Gitlab::Database::LoadBalancing.base_models.each do |model|
   # information.
   Gitlab::Database::LoadBalancing::Setup.new(model).setup
 
-  # We need re-rerun the setup when code reloads in development
-  Rails.application.reloader.to_prepare do
-    if Rails.env.development? || Rails.env.test?
-      Gitlab::Database::LoadBalancing::Setup.new(model).setup
-    end
-  end
-
   # Database queries may be run before we fork, so we must set up the load
   # balancer as early as possible. When we do fork, we need to make sure all the
   # hosts are disconnected.
diff --git a/config/initializers/memory_watchdog.rb b/config/initializers/memory_watchdog.rb
index 82ad615ce25..ce8e5029e7a 100644
--- a/config/initializers/memory_watchdog.rb
+++ b/config/initializers/memory_watchdog.rb
@@ -4,17 +4,33 @@ return unless Gitlab::Runtime.application?
 return unless Gitlab::Utils.to_boolean(ENV['GITLAB_MEMORY_WATCHDOG_ENABLED'])
 
 Gitlab::Cluster::LifecycleEvents.on_worker_start do
-  handler =
-    if Gitlab::Runtime.puma?
-      Gitlab::Memory::Watchdog::PumaHandler.new
-    elsif Gitlab::Runtime.sidekiq?
-      Gitlab::Memory::Watchdog::TermProcessHandler.new
-    else
-      Gitlab::Memory::Watchdog::NullHandler.instance
-    end
+  watchdog = Gitlab::Memory::Watchdog.new
+  max_strikes = ENV.fetch('GITLAB_MEMWD_MAX_STRIKES', 5).to_i
+  sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', 60).to_i
+  max_mem_growth = ENV.fetch('GITLAB_MEMWD_MAX_MEM_GROWTH', 3.0).to_f
+  max_heap_frag = ENV.fetch('GITLAB_MEMWD_MAX_HEAP_FRAG', 0.5).to_f
+
+  watchdog.configure do |config|
+    config.handler =
+      if Gitlab::Runtime.puma?
+        Gitlab::Memory::Watchdog::PumaHandler.new
+      elsif Gitlab::Runtime.sidekiq?
+        Gitlab::Memory::Watchdog::TermProcessHandler.new
+      else
+        Gitlab::Memory::Watchdog::NullHandler.instance
+      end
+
+    config.logger = Gitlab::AppLogger
+    config.sleep_time_seconds = sleep_time_seconds
+    # config.monitor.use MonitorClass, args*, max_strikes:, kwargs**, &block
+    config.monitors.use Gitlab::Memory::Watchdog::Monitor::HeapFragmentation,
+      max_heap_fragmentation: max_heap_frag,
+      max_strikes: max_strikes
+
+    config.monitors.use Gitlab::Memory::Watchdog::Monitor::UniqueMemoryGrowth,
+      max_mem_growth: max_mem_growth,
+      max_strikes: max_strikes
+  end
 
-  watchdog = Gitlab::Memory::Watchdog.new(
-    handler: handler, logger: Gitlab::AppLogger
-  )
   Gitlab::BackgroundTask.new(watchdog).start
 end
diff --git a/config/initializers/multi_json.rb b/config/initializers/multi_json.rb
index 5f98ce1961f..f382e2cb995 100644
--- a/config/initializers/multi_json.rb
+++ b/config/initializers/multi_json.rb
@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
 # Explicitly set the JSON adapter used by MultiJson
-# Currently we want this to default to the existing json gem
+#
+# This changes the default JSON adapter used by any gem dependencies
+# we have that rely on MultiJson for their JSON handling. We set this
+# to `oj` for a universal performance improvement in JSON handling
+# across those gems.
+
 MultiJson.use(:oj)
diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb
index 2454b5ea818..707e4efe7bf 100644
--- a/config/initializers/omniauth.rb
+++ b/config/initializers/omniauth.rb
@@ -11,23 +11,12 @@ if Gitlab::Auth::Ldap::Config.enabled?
   end
 end
 
-module OmniAuth
-  module Strategies
-    class AzureActivedirectoryV2
-      # override until https://github.com/RIPAGlobal/omniauth-azure-activedirectory-v2/pull/6 is merged
-      def callback_url
-        full_host + callback_path
-      end
-    end
-  end
-end
-
 OmniAuth.config.full_host = Gitlab::OmniauthInitializer.full_host
 
 OmniAuth.config.allowed_request_methods = [:post]
 # In case of auto sign-in, the GET method is used (users don't get to click on a button)
 OmniAuth.config.allowed_request_methods << :get if Gitlab.config.omniauth.auto_sign_in_with_provider.present?
-OmniAuth.config.before_request_phase do |env|
+OmniAuth.config.request_validation_phase do |env|
   Gitlab::RequestForgeryProtection.call(env)
 end
 
diff --git a/config/initializers/sawyer_patch.rb b/config/initializers/sawyer_patch.rb
index 08d249645cc..34d2843d165 100644
--- a/config/initializers/sawyer_patch.rb
+++ b/config/initializers/sawyer_patch.rb
@@ -24,6 +24,16 @@ module SawyerClassPatch
           end
         else
           define_method attribute do
+            Gitlab::Import::Logger.warn(
+              Gitlab::ApplicationContext.current.merge(
+                {
+                  message: 'Sawyer attribute called',
+                  attribute: attribute,
+                  caller: Gitlab::BacktraceCleaner.clean_backtrace(caller)
+                }
+              )
+            )
+
             @attrs[attribute.to_sym]
           end
 
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index 262ef1dea21..cef34425bf4 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -5,6 +5,16 @@ module SidekiqLogArguments
   end
 end
 
+def load_cron_jobs!
+  Sidekiq::Cron::Job.load_from_hash! Gitlab::SidekiqConfig.cron_jobs
+
+  Gitlab.ee do
+    Gitlab::Mirror.configure_cron_job!
+
+    Gitlab::Geo.configure_cron_jobs!
+  end
+end
+
 def enable_reliable_fetch?
   return true unless Feature::FlipperFeature.table_exists?
 
@@ -23,8 +33,6 @@ queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE
 
 enable_json_logs = Gitlab.config.sidekiq.log_format == 'json'
 enable_sidekiq_memory_killer = ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.nonzero?
-use_sidekiq_daemon_memory_killer = ENV.fetch("SIDEKIQ_DAEMON_MEMORY_KILLER", 1).to_i.nonzero?
-use_sidekiq_legacy_memory_killer = !use_sidekiq_daemon_memory_killer
 
 Sidekiq.configure_server do |config|
   config.options[:strict] = false
@@ -45,8 +53,7 @@ Sidekiq.configure_server do |config|
 
   config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator(
     metrics: Settings.monitoring.sidekiq_exporter,
-    arguments_logger: SidekiqLogArguments.enabled? && !enable_json_logs,
-    memory_killer: enable_sidekiq_memory_killer && use_sidekiq_legacy_memory_killer
+    arguments_logger: SidekiqLogArguments.enabled? && !enable_json_logs
   ))
 
   config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator)
@@ -62,7 +69,7 @@ Sidekiq.configure_server do |config|
     # To cancel job, it requires `SIDEKIQ_MONITOR_WORKER=1` to enable notification channel
     Gitlab::SidekiqDaemon::Monitor.instance.start
 
-    Gitlab::SidekiqDaemon::MemoryKiller.instance.start if enable_sidekiq_memory_killer && use_sidekiq_daemon_memory_killer
+    Gitlab::SidekiqDaemon::MemoryKiller.instance.start if enable_sidekiq_memory_killer
 
     first_sidekiq_worker = !ENV['SIDEKIQ_WORKER_ID'] || ENV['SIDEKIQ_WORKER_ID'] == '0'
     health_checks = Settings.monitoring.sidekiq_health_checks
@@ -81,30 +88,10 @@ Sidekiq.configure_server do |config|
     Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
   end
 
-  Gitlab.config.load_dynamic_cron_schedules!
-
-  # Sidekiq-cron: load recurring jobs from gitlab.yml
-  # UGLY Hack to get nested hash from settingslogic
-  cron_jobs = Gitlab::Json.parse(Gitlab.config.cron_jobs.to_json)
-  # UGLY hack: Settingslogic doesn't allow 'class' key
-  cron_jobs_required_keys = %w(job_class cron)
-  cron_jobs.each do |k, v|
-    if cron_jobs[k] && cron_jobs_required_keys.all? { |s| cron_jobs[k].key?(s) }
-      cron_jobs[k]['class'] = cron_jobs[k].delete('job_class')
-    else
-      cron_jobs.delete(k)
-      Gitlab::AppLogger.error("Invalid cron_jobs config key: '#{k}'. Check your gitlab config file.")
-    end
-  end
-  Sidekiq::Cron::Job.load_from_hash! cron_jobs
-
   Gitlab::SidekiqVersioning.install!
 
-  Gitlab.ee do
-    Gitlab::Mirror.configure_cron_job!
-
-    Gitlab::Geo.configure_cron_jobs!
-  end
+  config.options[:cron_poll_interval] = Gitlab.config.cron_jobs.poll_interval
+  load_cron_jobs!
 
   # Avoid autoload issue such as 'Mail::Parsers::AddressStruct'
   # https://github.com/mikel/mail/issues/912#issuecomment-214850355
@@ -127,4 +114,3 @@ end
 
 Sidekiq::Scheduled::Poller.prepend Gitlab::Patch::SidekiqPoller
 Sidekiq::Cron::Poller.prepend Gitlab::Patch::SidekiqPoller
-Sidekiq::Cron::Poller.prepend Gitlab::Patch::SidekiqCronPoller