Add latest changes from gitlab-org/gitlab@master

18 files changed, 226 insertions, 97 deletions

diff --git a/lib/gitlab/action_cable/request_store_callbacks.rb b/lib/gitlab/action_cable/request_store_callbacks.rb
index a9f30b0fc10..14d80a7c40c 100644
--- a/lib/gitlab/action_cable/request_store_callbacks.rb
+++ b/lib/gitlab/action_cable/request_store_callbacks.rb
@@ -5,8 +5,6 @@ module Gitlab
     module RequestStoreCallbacks
       def self.install
         ::ActionCable::Server::Worker.set_callback :work, :around, &wrapper
-        ::ActionCable::Channel::Base.set_callback :subscribe, :around, &wrapper
-        ::ActionCable::Channel::Base.set_callback :unsubscribe, :around, &wrapper
       end
 
       def self.wrapper
diff --git a/lib/gitlab/ci/components/header.rb b/lib/gitlab/ci/components/header.rb
deleted file mode 100644
index 732874d7a88..00000000000
--- a/lib/gitlab/ci/components/header.rb
+++ /dev/null
@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Ci
-    module Components
-      ##
-      # Components::Header class represents full component specification that is being prepended as first YAML document
-      # in the CI Component file.
-      #
-      class Header
-        attr_reader :errors
-
-        def initialize(header)
-          @header = header
-          @errors = []
-        end
-
-        def empty?
-          inputs_spec.to_h.empty?
-        end
-
-        def inputs(args)
-          @input ||= Ci::Input::Inputs.new(inputs_spec, args)
-        end
-
-        def context(args)
-          inputs(args).then do |input|
-            raise ArgumentError unless input.valid?
-
-            Ci::Interpolation::Context.new({ inputs: input.to_hash })
-          end
-        end
-
-        private
-
-        def inputs_spec
-          @header.dig(:spec, :inputs)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb
index 0b90d240a15..273d78bd583 100644
--- a/lib/gitlab/ci/config/external/file/artifact.rb
+++ b/lib/gitlab/ci/config/external/file/artifact.rb
@@ -22,7 +22,7 @@ module Gitlab
               strong_memoize(:content) do
                 Gitlab::Ci::ArtifactFileReader.new(artifact_job).read(location)
               rescue Gitlab::Ci::ArtifactFileReader::Error => error
-                errors.push(error.message)
+                errors.push(error.message) # TODO this memoizes the error message as a content!
               end
             end
 
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb
index 7060754a670..553f2a2d754 100644
--- a/lib/gitlab/ci/config/external/file/base.rb
+++ b/lib/gitlab/ci/config/external/file/base.rb
@@ -61,18 +61,6 @@ module Gitlab
               [params, context.project&.full_path, context.sha].hash
             end
 
-            def load_and_validate_expanded_hash!
-              context.logger.instrument(:config_file_fetch_content_hash) do
-                content_hash # calling the method loads then memoizes the result
-              end
-
-              context.logger.instrument(:config_file_expand_content_includes) do
-                expanded_content_hash # calling the method expands then memoizes the result
-              end
-
-              validate_hash!
-            end
-
             # This method is overridden to load context into the memoized result
             # or to lazily load context via BatchLoader
             def preload_context
@@ -94,32 +82,59 @@ module Gitlab
             end
 
             def validate_context!
-              raise NotImplementedError, 'subclass must implement validate_context'
+              raise NotImplementedError, 'subclass must implement `validate_context!`'
             end
 
             def validate_content!
-              if content.blank?
-                errors.push("Included file `#{masked_location}` is empty or does not exist!")
+              errors.push("Included file `#{masked_location}` is empty or does not exist!") if content.blank?
+            end
+
+            def load_and_validate_expanded_hash!
+              context.logger.instrument(:config_file_fetch_content_hash) do
+                content_result # calling the method loads YAML then memoizes the content result
+              end
+
+              context.logger.instrument(:config_file_interpolate_result) do
+                interpolator.interpolate!
+              end
+
+              return validate_interpolation! unless interpolator.valid?
+
+              context.logger.instrument(:config_file_expand_content_includes) do
+                expanded_content_hash # calling the method expands then memoizes the result
               end
+
+              validate_hash!
             end
 
             protected
 
             def content_result
-              strong_memoize(:content_hash) do
-                ::Gitlab::Ci::Config::Yaml
-                  .load_result!(content, project: context.project)
-              end
+              ::Gitlab::Ci::Config::Yaml
+                .load_result!(content, project: context.project)
+            end
+            strong_memoize_attr :content_result
+
+            def content_inputs
+              params.to_h[:with]
             end
+            strong_memoize_attr :content_inputs
 
             def content_hash
-              return unless content_result.valid?
+              interpolator.interpolate!
+
+              interpolator.to_hash
+            end
+            strong_memoize_attr :content_hash
 
-              content_result.content
+            def interpolator
+              External::Interpolator
+                .new(content_result, content_inputs, context)
             end
+            strong_memoize_attr :interpolator
 
             def expanded_content_hash
-              return unless content_hash
+              return if content_hash.blank?
 
               strong_memoize(:expanded_content_hash) do
                 expand_includes(content_hash)
@@ -132,6 +147,12 @@ module Gitlab
               end
             end
 
+            def validate_interpolation!
+              return if interpolator.valid?
+
+              errors.push("`#{masked_location}`: #{interpolator.error_message}")
+            end
+
             def expand_includes(hash)
               External::Processor.new(hash, context.mutate(expand_context_attrs)).perform
             end
diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb
index 7ab7dc3d64e..9679d78a1aa 100644
--- a/lib/gitlab/ci/config/external/file/component.rb
+++ b/lib/gitlab/ci/config/external/file/component.rb
@@ -11,6 +11,7 @@ module Gitlab
 
             def initialize(params, context)
               @location = params[:component]
+
               super
             end
 
@@ -48,9 +49,7 @@ module Gitlab
             end
 
             def validate_content!
-              return if content.present?
-
-              errors.push(component_result.message)
+              errors.push(component_result.message) unless content.present?
             end
 
             private
diff --git a/lib/gitlab/ci/config/external/interpolator.rb b/lib/gitlab/ci/config/external/interpolator.rb
new file mode 100644
index 00000000000..5629c4a9766
--- /dev/null
+++ b/lib/gitlab/ci/config/external/interpolator.rb
@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        ##
+        # Config::External::Interpolation perform includable file interpolation, and surfaces all possible interpolation
+        # errors. It is designed to provide an external file's validation context too.
+        #
+        class Interpolator
+          include ::Gitlab::Utils::StrongMemoize
+
+          attr_reader :config, :args, :ctx, :errors
+
+          def initialize(config, args, ctx = nil)
+            @config = config
+            @args = args.to_h
+            @ctx = ctx
+            @errors = []
+
+            validate!
+          end
+
+          def valid?
+            @errors.none?
+          end
+
+          def ready?
+            ##
+            # Interpolation is ready when it has been either interrupted by an error or finished with a result.
+            #
+            @result || @errors.any?
+          end
+
+          def interpolate?
+            enabled? && has_header? && valid?
+          end
+
+          def has_header?
+            config.has_header? && config.header.present?
+          end
+
+          def to_hash
+            @result.to_h
+          end
+
+          def error_message
+            # Interpolator can have multiple error messages, like: ["interpolation interrupted by errors", "unknown
+            # interpolation key: `abc`"] ?
+            #
+            # We are joining them together into a single one, because only one error can be surfaced when an external
+            # file gets included and is invalid. The limit to three error messages combined is more than required.
+            #
+            @errors.first(3).join(', ')
+          end
+
+          ##
+          # TODO Add `instrument.logger` instrumentation blocks:
+          #   https://gitlab.com/gitlab-org/gitlab/-/issues/396722
+          #
+          def interpolate!
+            return {} unless valid?
+            return @result ||= content.to_h unless interpolate?
+
+            return @errors.concat(header.errors) unless header.valid?
+            return @errors.concat(inputs.errors) unless inputs.valid?
+            return @errors.concat(context.errors) unless context.valid?
+            return @errors.concat(template.errors) unless template.valid?
+
+            @result ||= template.interpolated.to_h.deep_symbolize_keys
+          end
+          strong_memoize_attr :interpolate!
+
+          private
+
+          def validate!
+            return errors.push('content does not have a valid YAML syntax') unless config.valid?
+
+            return unless has_header? && !enabled?
+
+            errors.push('can not evaluate included file because interpolation is disabled')
+          end
+
+          def enabled?
+            return false if ctx.nil?
+
+            ::Feature.enabled?(:ci_includable_files_interpolation, ctx.project)
+          end
+
+          def header
+            @entry ||= Ci::Config::Header::Root.new(config.header).tap do |header|
+              header.key = 'header'
+
+              header.compose!
+            end
+          end
+
+          def content
+            @content ||= config.content
+          end
+
+          def spec
+            @spec ||= header.inputs_value
+          end
+
+          def inputs
+            @inputs ||= Ci::Input::Inputs.new(spec, args)
+          end
+
+          def context
+            @context ||= Ci::Interpolation::Context.new({ inputs: inputs.to_hash })
+          end
+
+          def template
+            @template ||= ::Gitlab::Ci::Interpolation::Template
+              .new(content, context)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/header/input.rb b/lib/gitlab/ci/config/header/input.rb
index 525b009afe3..7f0edaaac4c 100644
--- a/lib/gitlab/ci/config/header/input.rb
+++ b/lib/gitlab/ci/config/header/input.rb
@@ -6,6 +6,7 @@ module Gitlab
       module Header
         ##
         # Input parameter used for interpolation with the CI configuration.
+        #
         class Input < ::Gitlab::Config::Entry::Node
           include ::Gitlab::Config::Entry::Validatable
           include ::Gitlab::Config::Entry::Attributable
diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb
index 0b711f11db4..33f9a454106 100644
--- a/lib/gitlab/ci/config/yaml/result.rb
+++ b/lib/gitlab/ci/config/yaml/result.rb
@@ -17,6 +17,8 @@ module Gitlab
           end
 
           def has_header?
+            return false unless @config.first.is_a?(Hash)
+
             @config.size > 1 && @config.first.key?(:spec)
           end
 
diff --git a/lib/gitlab/ci/input/arguments/default.rb b/lib/gitlab/ci/input/arguments/default.rb
index fd61c1ab786..c6762b04870 100644
--- a/lib/gitlab/ci/input/arguments/default.rb
+++ b/lib/gitlab/ci/input/arguments/default.rb
@@ -9,7 +9,9 @@ module Gitlab
         #
         class Default < Input::Arguments::Base
           def validate!
-            error('invalid specification') unless default.present?
+            return error('argument specification invalid') unless spec.key?(:default)
+
+            error('invalid default value') unless default.is_a?(String) || default.nil?
           end
 
           ##
@@ -35,6 +37,8 @@ module Gitlab
           end
 
           def self.matches?(spec)
+            return false unless spec.is_a?(Hash)
+
             spec.count == 1 && spec.each_key.first == :default
           end
         end
diff --git a/lib/gitlab/ci/input/arguments/options.rb b/lib/gitlab/ci/input/arguments/options.rb
index debc89b10bd..855dab129be 100644
--- a/lib/gitlab/ci/input/arguments/options.rb
+++ b/lib/gitlab/ci/input/arguments/options.rb
@@ -25,7 +25,8 @@ module Gitlab
           # The configuration above will return an empty value.
           #
           def validate!
-            return error('argument specification invalid') if options.to_a.empty?
+            return error('argument specification invalid') unless options.is_a?(Array)
+            return error('options argument empty') if options.empty?
 
             if !value.nil?
               error("argument value #{value} not allowlisted") unless options.include?(value)
@@ -43,6 +44,8 @@ module Gitlab
           end
 
           def self.matches?(spec)
+            return false unless spec.is_a?(Hash)
+
             spec.count == 1 && spec.each_key.first == :options
           end
         end
diff --git a/lib/gitlab/ci/input/arguments/required.rb b/lib/gitlab/ci/input/arguments/required.rb
index b4e218ed29e..2e39f548731 100644
--- a/lib/gitlab/ci/input/arguments/required.rb
+++ b/lib/gitlab/ci/input/arguments/required.rb
@@ -28,7 +28,7 @@ module Gitlab
           #       website:
           # ```
           #
-          # An empty value, that has no specification is also considered as a "required" input, however we should
+          # An empty string value, that has no specification is also considered as a "required" input, however we should
           # never see that being used, because it will be rejected by Ci::Config::Header validation.
           #
           # ```yaml
@@ -36,8 +36,17 @@ module Gitlab
           #     inputs:
           #       website: ""
           # ```
+          #
+          # An empty hash value is also considered to be a required argument:
+          #
+          # ```yaml
+          #   spec:
+          #     inputs:
+          #       website: {}
+          # ```
+          #
           def self.matches?(spec)
-            spec.to_s.empty?
+            spec.blank?
           end
         end
       end
diff --git a/lib/gitlab/ci/input/inputs.rb b/lib/gitlab/ci/input/inputs.rb
index 743ae2ecf1e..1b544e63e7d 100644
--- a/lib/gitlab/ci/input/inputs.rb
+++ b/lib/gitlab/ci/input/inputs.rb
@@ -19,8 +19,8 @@ module Gitlab
         ].freeze
 
         def initialize(spec, args)
-          @spec = spec
-          @args = args
+          @spec = spec.to_h
+          @args = args.to_h
           @inputs = []
           @errors = []
 
diff --git a/lib/gitlab/ci/interpolation/access.rb b/lib/gitlab/ci/interpolation/access.rb
index 42598458902..f9bbd3e118d 100644
--- a/lib/gitlab/ci/interpolation/access.rb
+++ b/lib/gitlab/ci/interpolation/access.rb
@@ -45,7 +45,11 @@ module Gitlab
           raise ArgumentError, 'access path invalid' unless valid?
 
           @value ||= objects.inject(@ctx) do |memo, value|
-            memo.fetch(value.to_sym)
+            key = value.to_sym
+
+            break @errors.push("unknown interpolation key: `#{key}`") unless memo.key?(key)
+
+            memo.fetch(key)
           end
         rescue KeyError => e
           @errors.push(e)
diff --git a/lib/gitlab/ci/interpolation/context.rb b/lib/gitlab/ci/interpolation/context.rb
index ce7a86a3c9b..69c1fbb792c 100644
--- a/lib/gitlab/ci/interpolation/context.rb
+++ b/lib/gitlab/ci/interpolation/context.rb
@@ -38,6 +38,10 @@ module Gitlab
           @context.fetch(field)
         end
 
+        def key?(name)
+          @context.key?(name)
+        end
+
         def to_h
           @context.to_h
         end
@@ -53,7 +57,7 @@ module Gitlab
             end
           end
 
-          values.max
+          values.max.to_i
         end
 
         def self.fabricate(context)
diff --git a/lib/gitlab/database/load_balancing/action_cable_callbacks.rb b/lib/gitlab/database/load_balancing/action_cable_callbacks.rb
index 7164976ff73..fab691117ad 100644
--- a/lib/gitlab/database/load_balancing/action_cable_callbacks.rb
+++ b/lib/gitlab/database/load_balancing/action_cable_callbacks.rb
@@ -6,14 +6,10 @@ module Gitlab
       module ActionCableCallbacks
         def self.install
           ::ActionCable::Server::Worker.set_callback :work, :around, &wrapper
-          ::ActionCable::Channel::Base.set_callback :subscribe, :around, &wrapper
-          ::ActionCable::Channel::Base.set_callback :unsubscribe, :around, &wrapper
         end
 
         def self.wrapper
           lambda do |_, inner|
-            ::Gitlab::Database::LoadBalancing::Session.current.use_primary!
-
             inner.call
           ensure
             ::Gitlab::Database::LoadBalancing.release_hosts
diff --git a/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb b/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb
new file mode 100644
index 00000000000..a74d8982d73
--- /dev/null
+++ b/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Subscriptions
+      class ActionCableWithLoadBalancing < ::GraphQL::Subscriptions::ActionCableSubscriptions
+        extend ::Gitlab::Utils::Override
+
+        # When executing updates we are usually responding to a broadcast as a result of a DB update.
+        # We use the primary so that we are sure that we are returning the newly updated data.
+        override :execute_update
+        def execute_update(subscription_id, event, object)
+          ::Gitlab::Database::LoadBalancing::Session.current.use_primary!
+
+          super
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index e3756a8c9f6..10bb358a292 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -9,7 +9,6 @@ module Gitlab
 
         attach_to :active_record
 
-        IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze
         DB_COUNTERS = %i{count write_count cached_count}.freeze
         SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i.freeze
 
@@ -114,7 +113,7 @@ module Gitlab
         end
 
         def ignored_query?(payload)
-          payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql])
+          payload[:name] == 'SCHEMA' || payload[:name] == 'TRANSACTION'
         end
 
         def cached_query?(payload)
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 52b8d70c113..78bb526db53 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -370,16 +370,6 @@ module Gitlab
         }
       end
 
-      def merge_requests_users(time_period)
-        redis_usage_data do
-          Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(
-            event_names: :merge_request_action,
-            start_date: time_period[:created_at].first,
-            end_date: time_period[:created_at].last
-          )
-        end
-      end
-
       def installation_type
         if Rails.env.production?
           Gitlab::INSTALLATION_TYPE
@@ -447,9 +437,7 @@ module Gitlab
           projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))),
           remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id),
           snippets: distinct_count(::Snippet.where(time_period), :author_id)
-        }.tap do |h|
-          h[:merge_requests_users] = merge_requests_users(time_period) if time_period.present?
-        end
+        }
       end
       # rubocop: enable CodeReuse/ActiveRecord