Add latest changes from gitlab-org/gitlab@15-10-stable-eev15.10.0-rc42

11 files changed, 315 insertions, 44 deletions

diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 7c49b59a7f0..2390ba05916 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -164,7 +164,7 @@ module Gitlab
               artifacts: artifacts_value,
               release: release_value,
               after_script: after_script_value,
-              hooks: hooks_pre_get_sources_script_enabled? ? hooks_value : nil,
+              hooks: hooks_value,
               ignore: ignored?,
               allow_failure_criteria: allow_failure_criteria,
               needs: needs_defined? ? needs_value : nil,
@@ -194,10 +194,6 @@ module Gitlab
 
             allow_failure_value
           end
-
-          def hooks_pre_get_sources_script_enabled?
-            YamlProcessor::FeatureFlags.enabled?(:ci_hooks_pre_get_sources_script)
-          end
         end
       end
     end
diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb
index 6eef279d3de..156109a084d 100644
--- a/lib/gitlab/ci/config/external/context.rb
+++ b/lib/gitlab/ci/config/external/context.rb
@@ -9,29 +9,30 @@ module Gitlab
 
           TimeoutError = Class.new(StandardError)
 
-          MAX_INCLUDES = 100
-          NEW_MAX_INCLUDES = 150 # Update to MAX_INCLUDES when FF ci_includes_count_duplicates is removed
+          MAX_INCLUDES = 150
+          TEMP_MAX_INCLUDES = 100 # For logging; to be removed in https://gitlab.com/gitlab-org/gitlab/-/issues/367150
 
           include ::Gitlab::Utils::StrongMemoize
 
-          attr_reader :project, :sha, :user, :parent_pipeline, :variables
+          attr_reader :project, :sha, :user, :parent_pipeline, :variables, :pipeline_config
           attr_reader :expandset, :execution_deadline, :logger, :max_includes
 
           delegate :instrument, to: :logger
 
           def initialize(
             project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil,
-            logger: nil
+            pipeline_config: nil, logger: nil
           )
             @project = project
             @sha = sha
             @user = user
             @parent_pipeline = parent_pipeline
             @variables = variables || Ci::Variables::Collection.new
-            @expandset = Feature.enabled?(:ci_includes_count_duplicates, project) ? [] : Set.new
+            @pipeline_config = pipeline_config
+            @expandset = []
             @execution_deadline = 0
             @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project)
-            @max_includes = Feature.enabled?(:ci_includes_count_duplicates, project) ? NEW_MAX_INCLUDES : MAX_INCLUDES
+            @max_includes = MAX_INCLUDES
             yield self if block_given?
           end
 
@@ -91,6 +92,13 @@ module Gitlab
             expandset.map(&:metadata)
           end
 
+          # Some Ci::ProjectConfig sources prepend the config content with an "internal" `include`, which becomes
+          # the first included file. When running a pipeline, we pass pipeline_config into the context of the first
+          # included file, which we use in this method to determine if the file is an "internal" one.
+          def internal_include?
+            !!pipeline_config&.internal_include_prepended?
+          end
+
           protected
 
           attr_writer :expandset, :execution_deadline, :logger, :max_includes
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb
index 84f34f2584b..7060754a670 100644
--- a/lib/gitlab/ci/config/external/file/base.rb
+++ b/lib/gitlab/ci/config/external/file/base.rb
@@ -73,6 +73,18 @@ module Gitlab
               validate_hash!
             end
 
+            # This method is overridden to load context into the memoized result
+            # or to lazily load context via BatchLoader
+            def preload_context
+              # no-op
+            end
+
+            def preload_content
+              # calling the `content` method either loads content into the memoized result
+              # or lazily loads it via BatchLoader
+              content
+            end
+
             def validate_location!
               if invalid_location_type?
                 errors.push("Included file `#{masked_location}` needs to be a string")
@@ -93,6 +105,19 @@ module Gitlab
 
             protected
 
+            def content_result
+              strong_memoize(:content_hash) do
+                ::Gitlab::Ci::Config::Yaml
+                  .load_result!(content, project: context.project)
+              end
+            end
+
+            def content_hash
+              return unless content_result.valid?
+
+              content_result.content
+            end
+
             def expanded_content_hash
               return unless content_hash
 
@@ -101,14 +126,6 @@ module Gitlab
               end
             end
 
-            def content_hash
-              strong_memoize(:content_hash) do
-                ::Gitlab::Ci::Config::Yaml.load!(content)
-              end
-            rescue Gitlab::Config::Loader::FormatError
-              nil
-            end
-
             def validate_hash!
               if to_hash.blank?
                 errors.push("Included file `#{masked_location}` does not have valid YAML syntax!")
diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb
index 33e7724bf9b..7ab7dc3d64e 100644
--- a/lib/gitlab/ci/config/external/file/component.rb
+++ b/lib/gitlab/ci/config/external/file/component.rb
@@ -15,7 +15,7 @@ module Gitlab
             end
 
             def matching?
-              super && ::Feature.enabled?(:ci_include_components, context.project)
+              super && ::Feature.enabled?(:ci_include_components, context.project&.root_namespace)
             end
 
             def content
diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb
index f8d4cb27710..5efefeeaf9d 100644
--- a/lib/gitlab/ci/config/external/file/project.rb
+++ b/lib/gitlab/ci/config/external/file/project.rb
@@ -15,7 +15,8 @@ module Gitlab
               # `Repository#blobs_at` does not support files with the `/` prefix.
               @location = Gitlab::Utils.remove_leading_slashes(params[:file])
 
-              @project_name = get_project_name(params[:project])
+              # We are using the same downcase in the `project` method.
+              @project_name = get_project_name(params[:project]).to_s.downcase
               @ref_name = params[:ref] || 'HEAD'
 
               super
@@ -39,6 +40,15 @@ module Gitlab
               )
             end
 
+            def preload_context
+              #
+              # calling these methods lazily loads them via BatchLoader
+              #
+              project
+              can_access_local_content?
+              sha
+            end
+
             def validate_context!
               if !can_access_local_content?
                 errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.")
@@ -58,21 +68,48 @@ module Gitlab
             private
 
             def project
-              strong_memoize(:project) do
-                ::Project.find_by_full_path(project_name)
+              return legacy_project if ::Feature.disabled?(:ci_batch_project_includes_context, context.project)
+
+              # Although we use `where_full_path_in`, this BatchLoader does not reduce the number of queries to 1.
+              # That's because we use it in the `can_access_local_content?` and `sha` BatchLoaders
+              # as the `for` parameter. And this loads the project immediately.
+              BatchLoader.for(project_name)
+                         .batch do |project_names, loader|
+                ::Project.where_full_path_in(project_names.uniq).each do |project|
+                  # We are using the same downcase in the `initialize` method.
+                  loader.call(project.full_path.downcase, project)
+                end
               end
             end
 
             def can_access_local_content?
-              strong_memoize(:can_access_local_content) do
-                context.logger.instrument(:config_file_project_validate_access) do
-                  Ability.allowed?(context.user, :download_code, project)
+              if ::Feature.disabled?(:ci_batch_project_includes_context, context.project)
+                return legacy_can_access_local_content?
+              end
+
+              BatchLoader.for(project)
+                         .batch(key: context.user) do |projects, loader, args|
+                projects.uniq.each do |project|
+                  context.logger.instrument(:config_file_project_validate_access) do
+                    loader.call(project, Ability.allowed?(args[:key], :download_code, project))
+                  end
+                end
+              end
+            end
+
+            def sha
+              return legacy_sha if ::Feature.disabled?(:ci_batch_project_includes_context, context.project)
+
+              BatchLoader.for([project, ref_name])
+                         .batch do |project_ref_pairs, loader|
+                project_ref_pairs.uniq.each do |project, ref_name|
+                  loader.call([project, ref_name], project.commit(ref_name).try(:sha))
                 end
               end
             end
 
             def fetch_local_content
-              BatchLoader.for([sha, location])
+              BatchLoader.for([sha.to_s, location])
                          .batch(key: project) do |locations, loader, args|
                 context.logger.instrument(:config_file_fetch_project_content) do
                   args[:key].repository.blobs_at(locations).each do |blob|
@@ -84,8 +121,22 @@ module Gitlab
               end
             end
 
-            def sha
-              strong_memoize(:sha) do
+            def legacy_project
+              strong_memoize(:legacy_project) do
+                ::Project.find_by_full_path(project_name)
+              end
+            end
+
+            def legacy_can_access_local_content?
+              strong_memoize(:legacy_can_access_local_content) do
+                context.logger.instrument(:config_file_project_validate_access) do
+                  Ability.allowed?(context.user, :download_code, project)
+                end
+              end
+            end
+
+            def legacy_sha
+              strong_memoize(:legacy_sha) do
                 project.commit(ref_name).try(:sha)
               end
             end
@@ -94,7 +145,7 @@ module Gitlab
             def expand_context_attrs
               {
                 project: project,
-                sha: sha,
+                sha: sha.to_s, # we need to use `.to_s` to load the value from the BatchLoader
                 user: context.user,
                 parent_pipeline: context.parent_pipeline,
                 variables: context.variables
diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb
index 2982b0efb6c..7284d2a7e01 100644
--- a/lib/gitlab/ci/config/external/mapper/verifier.rb
+++ b/lib/gitlab/ci/config/external/mapper/verifier.rb
@@ -9,8 +9,56 @@ module Gitlab
           class Verifier < Base
             private
 
+            # rubocop: disable Metrics/CyclomaticComplexity
             def process_without_instrumentation(files)
+              if ::Feature.disabled?(:ci_batch_project_includes_context, context.project)
+                return legacy_process_without_instrumentation(files)
+              end
+
               files.each do |file|
+                if YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
+                  # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an
+                  # "internal" `include`. We use this condition to exclude that `include` from the included file set.
+                  context.expandset << file unless context.internal_include?
+                  verify_max_includes!
+                end
+
+                verify_execution_time!
+
+                file.validate_location!
+                file.preload_context if file.valid?
+              end
+
+              # We do not combine the loops because we need to load the context of all files via `BatchLoader`.
+              files.each do |file| # rubocop:disable Style/CombinableLoops
+                verify_execution_time!
+
+                file.validate_context! if file.valid?
+                file.preload_content if file.valid?
+              end
+
+              # We do not combine the loops because we need to load the content of all files via `BatchLoader`.
+              files.each do |file| # rubocop:disable Style/CombinableLoops
+                verify_max_includes! unless YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
+                verify_execution_time!
+
+                file.validate_content! if file.valid?
+                file.load_and_validate_expanded_hash! if file.valid?
+
+                context.expandset << file unless YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
+              end
+            end
+            # rubocop: enable Metrics/CyclomaticComplexity
+
+            def legacy_process_without_instrumentation(files)
+              files.each do |file|
+                if YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
+                  # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an
+                  # "internal" `include`. We use this condition to exclude that `include` from the included file set.
+                  context.expandset << file unless context.internal_include?
+                  verify_max_includes!
+                end
+
                 verify_execution_time!
 
                 file.validate_location!
@@ -21,23 +69,22 @@ module Gitlab
               # We do not combine the loops because we need to load the content of all files before continuing
               # to call `BatchLoader` for all locations.
               files.each do |file| # rubocop:disable Style/CombinableLoops
-                # Checking the max includes will be changed with https://gitlab.com/gitlab-org/gitlab/-/issues/367150
-                verify_max_includes!
+                verify_max_includes! unless YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
                 verify_execution_time!
 
                 file.validate_content! if file.valid?
                 file.load_and_validate_expanded_hash! if file.valid?
 
-                if context.expandset.is_a?(Array) # To be removed when FF 'ci_includes_count_duplicates' is removed
-                  context.expandset << file
-                else
-                  context.expandset.add(file)
-                end
+                context.expandset << file unless YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
               end
             end
 
             def verify_max_includes!
-              return if context.expandset.count < context.max_includes
+              if YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
+                return if context.expandset.count <= context.max_includes
+              else
+                return if context.expandset.count < context.max_includes # rubocop:disable Style/IfInsideElse
+              end
 
               raise Mapper::TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!"
             end
diff --git a/lib/gitlab/ci/config/header/input.rb b/lib/gitlab/ci/config/header/input.rb
new file mode 100644
index 00000000000..525b009afe3
--- /dev/null
+++ b/lib/gitlab/ci/config/header/input.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Header
+        ##
+        # Input parameter used for interpolation with the CI configuration.
+        class Input < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
+          include ::Gitlab::Config::Entry::Attributable
+
+          attributes :default, prefix: :input
+
+          validations do
+            validates :config, type: Hash, allowed_keys: [:default]
+            validates :key, alphanumeric: true
+            validates :input_default, alphanumeric: true, allow_nil: true
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/header/root.rb b/lib/gitlab/ci/config/header/root.rb
new file mode 100644
index 00000000000..251682d13b4
--- /dev/null
+++ b/lib/gitlab/ci/config/header/root.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Header
+        ##
+        # This class represents the root entry of the GitLab CI configuration header.
+        #
+        # A header is the first document in a multi-doc YAML that contains metadata
+        # and specifications about the GitLab CI configuration (the second document).
+        #
+        # The header is optional. A CI configuration can also be represented with a
+        # YAML containing a single document.
+        class Root < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Configurable
+
+          ALLOWED_KEYS = %i[spec].freeze
+
+          validations do
+            validates :config, type: Hash, allowed_keys: ALLOWED_KEYS
+          end
+
+          entry :spec, Header::Spec,
+            description: 'Specifications of the CI configuration.',
+            inherit: false,
+            default: {}
+
+          def inputs_value
+            spec_entry.inputs_value
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/header/spec.rb b/lib/gitlab/ci/config/header/spec.rb
new file mode 100644
index 00000000000..98d6d0d5783
--- /dev/null
+++ b/lib/gitlab/ci/config/header/spec.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Header
+        class Spec < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Configurable
+
+          ALLOWED_KEYS = %i[inputs].freeze
+
+          validations do
+            validates :config, type: Hash, allowed_keys: ALLOWED_KEYS
+          end
+
+          entry :inputs, ::Gitlab::Config::Entry::ComposableHash,
+            description: 'Allowed input parameters used for interpolation.',
+            inherit: false,
+            metadata: { composable_class: ::Gitlab::Ci::Config::Header::Input }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb
index 94ef0afe7f9..d1b1b8caa5c 100644
--- a/lib/gitlab/ci/config/yaml.rb
+++ b/lib/gitlab/ci/config/yaml.rb
@@ -7,23 +7,38 @@ module Gitlab
         AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze
         MAX_DOCUMENTS = 2
 
-        class << self
-          def load!(content)
+        class Loader
+          def initialize(content, project: nil)
+            @content = content
+            @project = project
+          end
+
+          def load!
             ensure_custom_tags
 
-            if ::Feature.enabled?(:ci_multi_doc_yaml)
-              Gitlab::Config::Loader::MultiDocYaml.new(
+            if project.present? && ::Feature.enabled?(:ci_multi_doc_yaml, project)
+              ::Gitlab::Config::Loader::MultiDocYaml.new(
                 content,
                 max_documents: MAX_DOCUMENTS,
                 additional_permitted_classes: AVAILABLE_TAGS
-              ).load!.first
+              ).load!
             else
-              Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load!
+              ::Gitlab::Config::Loader::Yaml
+                .new(content, additional_permitted_classes: AVAILABLE_TAGS)
+                .load!
             end
           end
 
+          def to_result
+            Yaml::Result.new(config: load!, error: nil)
+          rescue ::Gitlab::Config::Loader::FormatError => e
+            Yaml::Result.new(error: e)
+          end
+
           private
 
+          attr_reader :content, :project
+
           def ensure_custom_tags
             @ensure_custom_tags ||= begin
               AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) }
@@ -32,6 +47,23 @@ module Gitlab
             end
           end
         end
+
+        class << self
+          def load!(content, project: nil)
+            Loader.new(content, project: project).to_result.then do |result|
+              ##
+              # raise an error for backwards compatibility
+              #
+              raise result.error unless result.valid?
+
+              result.content
+            end
+          end
+
+          def load_result!(content, project: nil)
+            Loader.new(content, project: project).to_result
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb
new file mode 100644
index 00000000000..1a3ca53c161
--- /dev/null
+++ b/lib/gitlab/ci/config/yaml/result.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Yaml
+        class Result
+          attr_reader :error
+
+          def initialize(config: nil, error: nil)
+            @config = Array.wrap(config)
+            @error = error
+          end
+
+          def valid?
+            error.nil?
+          end
+
+          def has_header?
+            @config.size > 1
+          end
+
+          def header
+            raise ArgumentError unless has_header?
+
+            @config.first
+          end
+
+          def content
+            @config.last
+          end
+        end
+      end
+    end
+  end
+end