Add latest changes from gitlab-org/gitlab@master

11 files changed, 73 insertions, 29 deletions

diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb
index 96f5265ce23..833ce5e32fa 100644
--- a/lib/api/ci/helpers/runner.rb
+++ b/lib/api/ci/helpers/runner.rb
@@ -12,13 +12,13 @@ module API
         JOB_TOKEN_PARAM = :token
         LEGACY_SYSTEM_XID = '<legacy>'
 
-        def authenticate_runner!
+        def authenticate_runner!(update_contacted_at: true)
           track_runner_authentication
           forbidden! unless current_runner
 
           runner_details = get_runner_details_from_request
-          current_runner.heartbeat(runner_details)
-          current_runner_machine&.heartbeat(runner_details)
+          current_runner.heartbeat(runner_details, update_contacted_at: update_contacted_at)
+          current_runner_machine&.heartbeat(runner_details, update_contacted_at: update_contacted_at)
         end
 
         def get_runner_details_from_request
diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb
index 1c81db39bb1..0e67fb762a9 100644
--- a/lib/api/ci/runner.rb
+++ b/lib/api/ci/runner.rb
@@ -85,7 +85,11 @@ module API
           optional :system_id, type: String, desc: %q(The runner's system identifier)
         end
         post '/verify', urgency: :low, feature_category: :runner do
-          authenticate_runner!
+          # For runners that were created in the UI, we want to update the contacted_at value
+          # only when it starts polling for jobs
+          registering_created_runner = params[:token].start_with?(::Ci::Runner::CREATED_RUNNER_TOKEN_PREFIX)
+
+          authenticate_runner!(update_contacted_at: !registering_created_runner)
           status 200
 
           present current_runner, with: Entities::Ci::RunnerRegistrationDetails
diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb
index 881b131c252..156109a084d 100644
--- a/lib/gitlab/ci/config/external/context.rb
+++ b/lib/gitlab/ci/config/external/context.rb
@@ -92,10 +92,11 @@ module Gitlab
             expandset.map(&:metadata)
           end
 
-          # Some ProjectConfig sources inject an `include` into the config content. We use this
-          # method to exclude that `include` from the calculation of the total included files.
-          def contains_internal_include?
-            !!pipeline_config&.contains_internal_include?
+          # Some Ci::ProjectConfig sources prepend the config content with an "internal" `include`, which becomes
+          # the first included file. When running a pipeline, we pass pipeline_config into the context of the first
+          # included file, which we use in this method to determine if the file is an "internal" one.
+          def internal_include?
+            !!pipeline_config&.internal_include_prepended?
           end
 
           protected
diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb
index 6c2e52958da..7284d2a7e01 100644
--- a/lib/gitlab/ci/config/external/mapper/verifier.rb
+++ b/lib/gitlab/ci/config/external/mapper/verifier.rb
@@ -9,12 +9,20 @@ module Gitlab
           class Verifier < Base
             private
 
+            # rubocop: disable Metrics/CyclomaticComplexity
             def process_without_instrumentation(files)
               if ::Feature.disabled?(:ci_batch_project_includes_context, context.project)
                 return legacy_process_without_instrumentation(files)
               end
 
               files.each do |file|
+                if YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
+                  # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an
+                  # "internal" `include`. We use this condition to exclude that `include` from the included file set.
+                  context.expandset << file unless context.internal_include?
+                  verify_max_includes!
+                end
+
                 verify_execution_time!
 
                 file.validate_location!
@@ -31,19 +39,26 @@ module Gitlab
 
               # We do not combine the loops because we need to load the content of all files via `BatchLoader`.
               files.each do |file| # rubocop:disable Style/CombinableLoops
-                # Checking the max includes will be changed with https://gitlab.com/gitlab-org/gitlab/-/issues/367150
-                verify_max_includes!
+                verify_max_includes! unless YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
                 verify_execution_time!
 
                 file.validate_content! if file.valid?
                 file.load_and_validate_expanded_hash! if file.valid?
 
-                context.expandset << file
+                context.expandset << file unless YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
               end
             end
+            # rubocop: enable Metrics/CyclomaticComplexity
 
             def legacy_process_without_instrumentation(files)
               files.each do |file|
+                if YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
+                  # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an
+                  # "internal" `include`. We use this condition to exclude that `include` from the included file set.
+                  context.expandset << file unless context.internal_include?
+                  verify_max_includes!
+                end
+
                 verify_execution_time!
 
                 file.validate_location!
@@ -54,19 +69,22 @@ module Gitlab
               # We do not combine the loops because we need to load the content of all files before continuing
               # to call `BatchLoader` for all locations.
               files.each do |file| # rubocop:disable Style/CombinableLoops
-                # Checking the max includes will be changed with https://gitlab.com/gitlab-org/gitlab/-/issues/367150
-                verify_max_includes!
+                verify_max_includes! unless YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
                 verify_execution_time!
 
                 file.validate_content! if file.valid?
                 file.load_and_validate_expanded_hash! if file.valid?
 
-                context.expandset << file
+                context.expandset << file unless YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
               end
             end
 
             def verify_max_includes!
-              return if context.expandset.count < context.max_includes
+              if YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes)
+                return if context.expandset.count <= context.max_includes
+              else
+                return if context.expandset.count < context.max_includes # rubocop:disable Style/IfInsideElse
+              end
 
               raise Mapper::TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!"
             end
diff --git a/lib/gitlab/ci/project_config.rb b/lib/gitlab/ci/project_config.rb
index e69efb85a93..00b2ad58428 100644
--- a/lib/gitlab/ci/project_config.rb
+++ b/lib/gitlab/ci/project_config.rb
@@ -26,7 +26,7 @@ module Gitlab
       end
 
       delegate :content, :source, to: :@config, allow_nil: true
-      delegate :contains_internal_include?, to: :@config
+      delegate :internal_include_prepended?, to: :@config
 
       def exists?
         !!@config&.exists?
diff --git a/lib/gitlab/ci/project_config/auto_devops.rb b/lib/gitlab/ci/project_config/auto_devops.rb
index 70f8c7b9bb3..c5f010ebaea 100644
--- a/lib/gitlab/ci/project_config/auto_devops.rb
+++ b/lib/gitlab/ci/project_config/auto_devops.rb
@@ -13,7 +13,7 @@ module Gitlab
           end
         end
 
-        def contains_internal_include?
+        def internal_include_prepended?
           true
         end
 
diff --git a/lib/gitlab/ci/project_config/external_project.rb b/lib/gitlab/ci/project_config/external_project.rb
index b2982551458..0afdab23886 100644
--- a/lib/gitlab/ci/project_config/external_project.rb
+++ b/lib/gitlab/ci/project_config/external_project.rb
@@ -17,7 +17,7 @@ module Gitlab
           end
         end
 
-        def contains_internal_include?
+        def internal_include_prepended?
           true
         end
 
diff --git a/lib/gitlab/ci/project_config/remote.rb b/lib/gitlab/ci/project_config/remote.rb
index 17d5a9ee68f..19cbf8e9c1e 100644
--- a/lib/gitlab/ci/project_config/remote.rb
+++ b/lib/gitlab/ci/project_config/remote.rb
@@ -12,7 +12,7 @@ module Gitlab
           end
         end
 
-        def contains_internal_include?
+        def internal_include_prepended?
           true
         end
 
diff --git a/lib/gitlab/ci/project_config/repository.rb b/lib/gitlab/ci/project_config/repository.rb
index c975023d35e..272425fd546 100644
--- a/lib/gitlab/ci/project_config/repository.rb
+++ b/lib/gitlab/ci/project_config/repository.rb
@@ -12,7 +12,7 @@ module Gitlab
           end
         end
 
-        def contains_internal_include?
+        def internal_include_prepended?
           true
         end
 
diff --git a/lib/gitlab/ci/project_config/source.rb b/lib/gitlab/ci/project_config/source.rb
index 30edf1b552a..9a4a6394fa1 100644
--- a/lib/gitlab/ci/project_config/source.rb
+++ b/lib/gitlab/ci/project_config/source.rb
@@ -24,8 +24,8 @@ module Gitlab
           raise NotImplementedError
         end
 
-        # Indicates if we are internally injecting an 'include' into the content
-        def contains_internal_include?
+        # Indicates if we are prepending the content with an "internal" `include`
+        def internal_include_prepended?
           false
         end
 
diff --git a/lib/gitlab/email/html_to_markdown_parser.rb b/lib/gitlab/email/html_to_markdown_parser.rb
index 42dd012308b..5dd3725cc3e 100644
--- a/lib/gitlab/email/html_to_markdown_parser.rb
+++ b/lib/gitlab/email/html_to_markdown_parser.rb
@@ -5,25 +5,46 @@ require 'nokogiri'
 module Gitlab
   module Email
     class HtmlToMarkdownParser < Html2Text
-      ADDITIONAL_TAGS = %w[em strong img details].freeze
-      IMG_ATTRS = %w[alt src].freeze
+      extend Gitlab::Utils::Override
 
+      # List of tags to be converted by Markdown.
+      #
+      # All attributes are removed except for the defined ones.
+      #
+      #   <tag> => [<attribute to keep>, ...]
+      ALLOWED_TAG_ATTRIBUTES = {
+        'em' => [],
+        'strong' => [],
+        'details' => [],
+        'img' => %w[alt src]
+      }.freeze
+      private_constant :ALLOWED_TAG_ATTRIBUTES
+
+      # This redefinition can be removed once https://github.com/soundasleep/html2text_ruby/pull/30
+      # is merged and released.
       def self.convert(html)
         html = fix_newlines(replace_entities(html))
         doc = Nokogiri::HTML(html)
 
-        HtmlToMarkdownParser.new(doc).convert
+        new(doc).convert
       end
 
+      private
+
+      override :iterate_over
       def iterate_over(node)
-        return super unless ADDITIONAL_TAGS.include?(node.name)
+        allowed_attributes = ALLOWED_TAG_ATTRIBUTES[node.name]
+        return super unless allowed_attributes
 
-        if node.name == 'img'
-          node.keys.each { |key| node.remove_attribute(key) unless IMG_ATTRS.include?(key) } # rubocop:disable Style/HashEachMethods
-        end
+        remove_attributes(node, allowed_attributes)
 
         Kramdown::Document.new(node.to_html, input: 'html').to_commonmark
       end
+
+      def remove_attributes(node, allowed_attributes)
+        to_remove = (node.keys - allowed_attributes)
+        to_remove.each { |key| node.remove_attribute(key) }
+      end
     end
   end
 end