Add latest changes from gitlab-org/gitlab@13-6-stable-eev13.6.0-rc42

11 files changed, 138 insertions, 29 deletions

diff --git a/config/initializers/0_inject_feature_flags.rb b/config/initializers/0_inject_feature_flags.rb
index 5b33b3bb4ea..74470a6cbfc 100644
--- a/config/initializers/0_inject_feature_flags.rb
+++ b/config/initializers/0_inject_feature_flags.rb
@@ -4,3 +4,42 @@
 Feature.register_feature_groups
 Feature.register_definitions
 Feature.register_hot_reloader unless Rails.configuration.cache_classes
+
+# This disallows usage of licensed feature names with the same name
+# as feature flags. This naming collision creates confusion and it was
+# decided to be removed in favor of explicit check.
+# https://gitlab.com/gitlab-org/gitlab/-/issues/259611
+if Gitlab.ee? && Gitlab.dev_or_test_env?
+  # These are the names of feature flags that do violate the constraint of
+  # being unique to licensed names. These feature flags should be reworked to
+  # be "development" with explicit check
+  IGNORED_FEATURE_FLAGS = %i[
+    swimlanes
+  ].to_set
+
+  # First, we validate a list of overrides to ensure that these overrides
+  # are removed if feature flag is gone
+  missing_feature_flags = IGNORED_FEATURE_FLAGS.reject do |feature_flag|
+    Feature::Definition.definitions[feature_flag]
+  end
+
+  if missing_feature_flags.any?
+    raise "The following feature flags were added as an override for discovering licensed features. " \
+          "Since these feature flags seems to be gone, ensure to remove them from \`IGNORED_FEATURE_FLAGS\` " \
+          "in \`#{__FILE__}'`: #{missing_feature_flags.join(", ")}"
+  end
+
+  # Second, we validate that there's no feature flag under the name as licensed feature
+  # flag, to ensure that the name used, is unique
+  licensed_features = License::PLANS_BY_FEATURE.keys.select do |licensed_feature_name|
+    IGNORED_FEATURE_FLAGS.exclude?(licensed_feature_name) &&
+      Feature::Definition.definitions[licensed_feature_name]
+  end
+
+  if licensed_features.any?
+    raise "The following feature flags do use a licensed feature. " \
+          "To avoid the confusion between their usage it is disallowed to use feature flag " \
+          "with exact the same name as licensed feature name. Use a different name to create " \
+          "a distinction: #{licensed_features.join(", ")}"
+  end
+end
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index affbc85d5a9..022f372a608 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -198,7 +198,7 @@ Settings.gitlab.default_projects_features['snippets']           = true if Settin
 Settings.gitlab.default_projects_features['builds']             = true if Settings.gitlab.default_projects_features['builds'].nil?
 Settings.gitlab.default_projects_features['container_registry'] = true if Settings.gitlab.default_projects_features['container_registry'].nil?
 Settings.gitlab.default_projects_features['visibility_level']   = Settings.__send__(:verify_constant, Gitlab::VisibilityLevel, Settings.gitlab.default_projects_features['visibility_level'], Gitlab::VisibilityLevel::PRIVATE)
-Settings.gitlab['domain_whitelist'] ||= []
+Settings.gitlab['domain_allowlist'] ||= []
 Settings.gitlab['import_sources'] ||= Gitlab::ImportSources.values
 Settings.gitlab['trusted_proxies'] ||= []
 Settings.gitlab['content_security_policy'] ||= Gitlab::ContentSecurityPolicy::ConfigLoader.default_settings_hash
@@ -354,25 +354,24 @@ Settings.uploads['object_store']['remote_directory'] ||= 'uploads'
 # Packages
 #
 Settings['packages'] ||= Settingslogic.new({})
-Settings.packages['enabled']      = true if Settings.packages['enabled'].nil?
-Settings.packages['storage_path'] = Settings.absolute(Settings.packages['storage_path'] || File.join(Settings.shared['path'], "packages"))
-Settings.packages['object_store'] = ObjectStoreSettings.legacy_parse(Settings.packages['object_store'])
+Settings.packages['enabled']       = true if Settings.packages['enabled'].nil?
+Settings.packages['dpkg_deb_path'] = '/usr/bin/dpkg-deb' if Settings.packages['dpkg_deb_path'].nil?
+Settings.packages['storage_path']  = Settings.absolute(Settings.packages['storage_path'] || File.join(Settings.shared['path'], "packages"))
+Settings.packages['object_store']  = ObjectStoreSettings.legacy_parse(Settings.packages['object_store'])
 
 #
 # Dependency Proxy
 #
-Gitlab.ee do
-  Settings['dependency_proxy'] ||= Settingslogic.new({})
-  Settings.dependency_proxy['enabled']      = true if Settings.dependency_proxy['enabled'].nil?
-  Settings.dependency_proxy['storage_path'] = Settings.absolute(Settings.dependency_proxy['storage_path'] || File.join(Settings.shared['path'], "dependency_proxy"))
-  Settings.dependency_proxy['object_store'] = ObjectStoreSettings.legacy_parse(Settings.dependency_proxy['object_store'])
-
-  # For first iteration dependency proxy uses Rails server to download blobs.
-  # To ensure acceptable performance we only allow feature to be used with
-  # multithreaded web-server Puma. This will be removed once download logic is moved
-  # to GitLab workhorse
-  Settings.dependency_proxy['enabled'] = false unless Gitlab::Runtime.puma?
-end
+Settings['dependency_proxy'] ||= Settingslogic.new({})
+Settings.dependency_proxy['enabled']      = true if Settings.dependency_proxy['enabled'].nil?
+Settings.dependency_proxy['storage_path'] = Settings.absolute(Settings.dependency_proxy['storage_path'] || File.join(Settings.shared['path'], "dependency_proxy"))
+Settings.dependency_proxy['object_store'] = ObjectStoreSettings.legacy_parse(Settings.dependency_proxy['object_store'])
+
+# For first iteration dependency proxy uses Rails server to download blobs.
+# To ensure acceptable performance we only allow feature to be used with
+# multithreaded web-server Puma. This will be removed once download logic is moved
+# to GitLab workhorse
+Settings.dependency_proxy['enabled'] = false unless Gitlab::Runtime.puma?
 
 #
 # Terraform state
@@ -414,7 +413,7 @@ Settings.cron_jobs['pipeline_schedule_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['pipeline_schedule_worker']['cron'] ||= '19 * * * *'
 Settings.cron_jobs['pipeline_schedule_worker']['job_class'] = 'PipelineScheduleWorker'
 Settings.cron_jobs['expire_build_artifacts_worker'] ||= Settingslogic.new({})
-Settings.cron_jobs['expire_build_artifacts_worker']['cron'] ||= '50 * * * *'
+Settings.cron_jobs['expire_build_artifacts_worker']['cron'] ||= '*/7 * * * *'
 Settings.cron_jobs['expire_build_artifacts_worker']['job_class'] = 'ExpireBuildArtifactsWorker'
 Settings.cron_jobs['ci_schedule_delete_objects_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['ci_schedule_delete_objects_worker']['cron'] ||= '*/16 * * * *'
@@ -530,6 +529,9 @@ Settings.cron_jobs['analytics_instance_statistics_count_job_trigger_worker']['jo
 Settings.cron_jobs['member_invitation_reminder_emails_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['member_invitation_reminder_emails_worker']['cron'] ||= '0 0 * * *'
 Settings.cron_jobs['member_invitation_reminder_emails_worker']['job_class'] = 'MemberInvitationReminderEmailsWorker'
+Settings.cron_jobs['schedule_merge_request_cleanup_refs_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['schedule_merge_request_cleanup_refs_worker']['cron'] ||= '* * * * *'
+Settings.cron_jobs['schedule_merge_request_cleanup_refs_worker']['job_class'] = 'ScheduleMergeRequestCleanupRefsWorker'
 
 Gitlab.ee do
   Settings.cron_jobs['active_user_count_threshold_worker'] ||= Settingslogic.new({})
@@ -604,6 +606,9 @@ Gitlab.ee do
   Settings.cron_jobs['elastic_remove_expired_namespace_subscriptions_from_index_cron_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['elastic_remove_expired_namespace_subscriptions_from_index_cron_worker']['cron'] ||= '10 3 * * *'
   Settings.cron_jobs['elastic_remove_expired_namespace_subscriptions_from_index_cron_worker']['job_class'] ||= 'ElasticRemoveExpiredNamespaceSubscriptionsFromIndexCronWorker'
+  Settings.cron_jobs['elastic_migration_worker'] ||= Settingslogic.new({})
+  Settings.cron_jobs['elastic_migration_worker']['cron'] ||= '*/30 * * * *'
+  Settings.cron_jobs['elastic_migration_worker']['job_class'] ||= 'Elastic::MigrationWorker'
   Settings.cron_jobs['sync_seat_link_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['sync_seat_link_worker']['cron'] ||= "#{rand(60)} 0 * * *"
   Settings.cron_jobs['sync_seat_link_worker']['job_class'] = 'SyncSeatLinkWorker'
diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb
index dbaebc83658..65ff6b656b9 100644
--- a/config/initializers/7_prometheus_metrics.rb
+++ b/config/initializers/7_prometheus_metrics.rb
@@ -70,7 +70,7 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
     Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION }, 1)
 
     unless Gitlab::Runtime.sidekiq?
-      Gitlab::Metrics::RequestsRackMiddleware.initialize_http_request_duration_seconds
+      Gitlab::Metrics::RequestsRackMiddleware.initialize_metrics
     end
   rescue IOError => e
     Gitlab::ErrorTracking.track_exception(e)
diff --git a/config/initializers/console_message.rb b/config/initializers/console_message.rb
index 523a3898043..fe47195062b 100644
--- a/config/initializers/console_message.rb
+++ b/config/initializers/console_message.rb
@@ -4,6 +4,7 @@ if Gitlab::Runtime.console?
   justify = 15
 
   puts '-' * 80
+  puts " Ruby:".ljust(justify) + RUBY_DESCRIPTION
   puts " GitLab:".ljust(justify) + "#{Gitlab::VERSION} (#{Gitlab.revision}) #{Gitlab.ee? ? 'EE' : 'FOSS'}"
   puts " GitLab Shell:".ljust(justify) + "#{Gitlab::VersionInfo.parse(Gitlab::Shell.version)}"
 
@@ -19,4 +20,15 @@ if Gitlab::Runtime.console?
   end
 
   puts '-' * 80
+
+  # Stop irb from writing a history file by default.
+  module IrbNoHistory
+    def init_config(*)
+      super
+
+      IRB.conf[:SAVE_HISTORY] = false
+    end
+  end
+
+  IRB.singleton_class.prepend(IrbNoHistory)
 end
diff --git a/config/initializers/grape_validators.rb b/config/initializers/grape_validators.rb
index 22f2c9ecf92..715949a276f 100644
--- a/config/initializers/grape_validators.rb
+++ b/config/initializers/grape_validators.rb
@@ -8,3 +8,4 @@ Grape::Validations.register_validator(:integer_none_any, ::API::Validations::Val
 Grape::Validations.register_validator(:array_none_any, ::API::Validations::Validators::ArrayNoneAny)
 Grape::Validations.register_validator(:check_assignees_count, ::API::Validations::Validators::CheckAssigneesCount)
 Grape::Validations.register_validator(:untrusted_regexp, ::API::Validations::Validators::UntrustedRegexp)
+Grape::Validations.register_validator(:email_or_email_list, ::API::Validations::Validators::EmailOrEmailList)
diff --git a/config/initializers/labkit_middleware.rb b/config/initializers/labkit_middleware.rb
index ea4103f052f..748666b6cd7 100644
--- a/config/initializers/labkit_middleware.rb
+++ b/config/initializers/labkit_middleware.rb
@@ -1,3 +1,36 @@
 # frozen_string_literal: true
 
-Rails.application.config.middleware.use(Labkit::Middleware::Rack)
+# partial backport of https://github.com/rails/rails/pull/38169
+# this is in order to be able to re-order rack middlewares.
+
+if ActionDispatch::MiddlewareStack.method_defined?(:move)
+  warn "`move` is now defined in in ActionDispatch itself: https://github.com/rails/rails/pull/38169, please remove this patch from #{__FILE__}"
+else
+  module ActionDispatch
+    class MiddlewareStack
+      def move(target, source)
+        source_index = assert_index(source, :before)
+        source_middleware = middlewares.delete_at(source_index)
+
+        target_index = assert_index(target, :before)
+        middlewares.insert(target_index, source_middleware)
+      end
+    end
+  end
+end
+
+unless Rails::Configuration::MiddlewareStackProxy.method_defined?(:move)
+  module Rails
+    module Configuration
+      class MiddlewareStackProxy
+        def move(*args, &block)
+          @operations << ->(middleware) { middleware.send(__method__, *args, &block) }
+        end
+        ruby2_keywords(:move) if respond_to?(:ruby2_keywords, true)
+      end
+    end
+  end
+end
+
+Rails.application.config.middleware.move(1, ActionDispatch::RequestId)
+Rails.application.config.middleware.insert_after(ActionDispatch::RequestId, Labkit::Middleware::Rack)
diff --git a/config/initializers/oj.rb b/config/initializers/oj.rb
deleted file mode 100644
index 3fa26259fc6..00000000000
--- a/config/initializers/oj.rb
+++ /dev/null
@@ -1,4 +0,0 @@
-# frozen_string_literal: true
-
-# Ensure Oj runs in json-gem compatibility mode by default
-Oj.default_options = { mode: :rails }
diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb
index b0778633199..58bf3f6013c 100644
--- a/config/initializers/rack_attack.rb
+++ b/config/initializers/rack_attack.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Specs for this file can be found on:
 # * spec/lib/gitlab/throttle_spec.rb
 # * spec/requests/rack_attack_global_spec.rb
@@ -15,6 +17,13 @@ module Gitlab::Throttle
     Rack::Attack.throttles.key?('protected paths')
   end
 
+  def self.bypass_header
+    env_value = ENV['GITLAB_THROTTLE_BYPASS_HEADER']
+    return unless env_value.present?
+
+    "HTTP_#{env_value.upcase.tr('-', '_')}"
+  end
+
   def self.unauthenticated_options
     limit_proc = proc { |req| settings.throttle_unauthenticated_requests_per_period }
     period_proc = proc { |req| settings.throttle_unauthenticated_period_in_seconds.seconds }
@@ -112,6 +121,11 @@ class Rack::Attack
     end
   end
 
+  safelist('throttle_bypass_header') do |req|
+    Gitlab::Throttle.bypass_header.present? &&
+      req.get_header(Gitlab::Throttle.bypass_header) == '1'
+  end
+
   class Request
     def unauthenticated?
       !(authenticated_user_id([:api, :rss, :ics]) || authenticated_runner_id)
diff --git a/config/initializers/rack_attack_logging.rb b/config/initializers/rack_attack_logging.rb
index a95cb09755b..e89c6b1b794 100644
--- a/config/initializers/rack_attack_logging.rb
+++ b/config/initializers/rack_attack_logging.rb
@@ -5,13 +5,15 @@
 ActiveSupport::Notifications.subscribe(/rack_attack/) do |name, start, finish, request_id, payload|
   req = payload[:request]
 
-  if [:throttle, :blocklist].include? req.env['rack.attack.match_type']
+  case req.env['rack.attack.match_type']
+  when :throttle, :blocklist
     rack_attack_info = {
       message: 'Rack_Attack',
       env: req.env['rack.attack.match_type'],
       remote_ip: req.ip,
       request_method: req.request_method,
-      path: req.fullpath
+      path: req.fullpath,
+      matched: req.env['rack.attack.matched']
     }
 
     throttles_with_user_information = [
@@ -25,11 +27,12 @@ ActiveSupport::Notifications.subscribe(/rack_attack/) do |name, start, finish, r
       user_id = req.env['rack.attack.match_discriminator']
       user = User.find_by(id: user_id)
 
-      rack_attack_info[:throttle_type] = req.env['rack.attack.matched']
       rack_attack_info[:user_id] = user_id
-      rack_attack_info[:username] = user.username unless user.nil?
+      rack_attack_info['meta.user'] = user.username unless user.nil?
     end
 
     Gitlab::AuthLogger.error(rack_attack_info)
+  when :safelist
+    Gitlab::Instrumentation::Throttle.safelist = req.env['rack.attack.matched']
   end
 end
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index 72e2b94fe07..8e3241a2e4c 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -1,4 +1,9 @@
 # frozen_string_literal: true
+module SidekiqLogArguments
+  def self.enabled?
+    Gitlab::Utils.to_boolean(ENV['SIDEKIQ_LOG_ARGUMENTS'], default: true)
+  end
+end
 
 def enable_reliable_fetch?
   return true unless Feature::FlipperFeature.table_exists?
@@ -35,7 +40,7 @@ Sidekiq.configure_server do |config|
 
   config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator({
     metrics: Settings.monitoring.sidekiq_exporter,
-    arguments_logger: ENV['SIDEKIQ_LOG_ARGUMENTS'] && !enable_json_logs,
+    arguments_logger: SidekiqLogArguments.enabled? && !enable_json_logs,
     memory_killer: enable_sidekiq_memory_killer && use_sidekiq_legacy_memory_killer
   }))
 
diff --git a/config/initializers/tracing.rb b/config/initializers/tracing.rb
index f26fb18f3ea..3058bdeb84e 100644
--- a/config/initializers/tracing.rb
+++ b/config/initializers/tracing.rb
@@ -2,7 +2,8 @@
 
 if Labkit::Tracing.enabled?
   Rails.application.configure do |config|
-    config.middleware.insert_after Labkit::Middleware::Rack, ::Labkit::Tracing::RackMiddleware
+    # Rack needs to parse multipart messages before ActionDispatch can filter parameters
+    config.middleware.insert_after Rack::MethodOverride, ::Labkit::Tracing::RackMiddleware
   end
 
   # Instrument Redis