Add latest changes from gitlab-org/gitlab@13-9-stable-eev13.9.0-rc42

10 files changed, 218 insertions, 14 deletions

diff --git a/lib/gitlab/ci/config/entry/commands.rb b/lib/gitlab/ci/config/entry/commands.rb
index 7a86fca3056..341f87b44ab 100644
--- a/lib/gitlab/ci/config/entry/commands.rb
+++ b/lib/gitlab/ci/config/entry/commands.rb
@@ -10,12 +10,14 @@ module Gitlab
         class Commands < ::Gitlab::Config::Entry::Node
           include ::Gitlab::Config::Entry::Validatable
 
+          MAX_NESTING_LEVEL = 10
+
           validations do
-            validates :config, string_or_nested_array_of_strings: true
+            validates :config, string_or_nested_array_of_strings: { max_level: MAX_NESTING_LEVEL }
           end
 
           def value
-            Array(@config).flatten(1)
+            Array(@config).flatten(MAX_NESTING_LEVEL)
           end
         end
       end
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 85e3514499c..a20b802be58 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -14,7 +14,7 @@ module Gitlab
           ALLOWED_KEYS = %i[tags script type image services start_in artifacts
                             cache dependencies before_script after_script
                             environment coverage retry parallel interruptible timeout
-                            resource_group release secrets].freeze
+                            release secrets].freeze
 
           REQUIRED_BY_NEEDS = %i[stage].freeze
 
@@ -30,7 +30,6 @@ module Gitlab
               }
 
               validates :dependencies, array_of_strings: true
-              validates :resource_group, type: String
               validates :allow_failure, hash_or_boolean: true
             end
 
@@ -124,7 +123,7 @@ module Gitlab
 
           attributes :script, :tags, :when, :dependencies,
                      :needs, :retry, :parallel, :start_in,
-                     :interruptible, :timeout, :resource_group,
+                     :interruptible, :timeout,
                      :release, :allow_failure
 
           def self.matching?(name, config)
@@ -174,7 +173,6 @@ module Gitlab
               ignore: ignored?,
               allow_failure_criteria: allow_failure_criteria,
               needs: needs_defined? ? needs_value : nil,
-              resource_group: resource_group,
               scheduling_type: needs_defined? ? :dag : :stage
             ).compact
           end
@@ -186,8 +184,6 @@ module Gitlab
           private
 
           def allow_failure_criteria
-            return unless ::Gitlab::Ci::Features.allow_failure_with_exit_codes_enabled?
-
             if allow_failure_defined? && allow_failure_value.is_a?(Hash)
               allow_failure_value
             end
diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb
index 5ef8cfbddb7..9584d19bdec 100644
--- a/lib/gitlab/ci/config/entry/processable.rb
+++ b/lib/gitlab/ci/config/entry/processable.rb
@@ -15,7 +15,7 @@ module Gitlab
           include ::Gitlab::Config::Entry::Inheritable
 
           PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables
-                                        inherit allow_failure when needs].freeze
+                                        inherit allow_failure when needs resource_group].freeze
 
           included do
             validations do
@@ -32,6 +32,7 @@ module Gitlab
               with_options allow_nil: true do
                 validates :extends, array_of_strings_or_string: true
                 validates :rules, array_of_hashes: true
+                validates :resource_group, type: String
               end
             end
 
@@ -64,7 +65,7 @@ module Gitlab
               inherit: false,
               default: {}
 
-            attributes :extends, :rules
+            attributes :extends, :rules, :resource_group
           end
 
           def compose!(deps = nil)
@@ -125,7 +126,8 @@ module Gitlab
               rules: rules_value,
               variables: root_and_job_variables_value,
               only: only_value,
-              except: except_value }.compact
+              except: except_value,
+              resource_group: resource_group }.compact
           end
 
           def root_and_job_variables_value
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb
index 4684a9eb981..7d3fddd850d 100644
--- a/lib/gitlab/ci/config/external/file/base.rb
+++ b/lib/gitlab/ci/config/external/file/base.rb
@@ -60,7 +60,7 @@ module Gitlab
 
             def content_hash
               strong_memoize(:content_yaml) do
-                Gitlab::Config::Loader::Yaml.new(content).load!
+                ::Gitlab::Ci::Config::Yaml.load!(content)
               end
             rescue Gitlab::Config::Loader::FormatError
               nil
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index 4d91cfd4c57..b85b7a9edeb 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -99,8 +99,6 @@ module Gitlab
           end
 
           def expand_variables(data)
-            return data unless ::Feature.enabled?(:variables_in_include_section_ci)
-
             if data.is_a?(String)
               expand(data)
             else
diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb
new file mode 100644
index 00000000000..de833619c8d
--- /dev/null
+++ b/lib/gitlab/ci/config/yaml.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Yaml
+        AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze
+
+        class << self
+          def load!(content)
+            ensure_custom_tags
+
+            Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load!
+          end
+
+          private
+
+          def ensure_custom_tags
+            @ensure_custom_tags ||= begin
+              AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) }
+
+              true
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/yaml/tags.rb b/lib/gitlab/ci/config/yaml/tags.rb
new file mode 100644
index 00000000000..1575edad3b0
--- /dev/null
+++ b/lib/gitlab/ci/config/yaml/tags.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Yaml
+        module Tags
+          TagError = Class.new(StandardError)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/yaml/tags/base.rb b/lib/gitlab/ci/config/yaml/tags/base.rb
new file mode 100644
index 00000000000..13416a4afb6
--- /dev/null
+++ b/lib/gitlab/ci/config/yaml/tags/base.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Yaml
+        module Tags
+          class Base
+            CircularReferenceError = Class.new(Tags::TagError)
+            NotValidError = Class.new(Tags::TagError)
+
+            extend ::Gitlab::Utils::Override
+
+            attr_accessor :resolved_status, :resolved_value, :data
+
+            def self.tag
+              raise NotImplementedError
+            end
+
+            # Only one of the `seq`, `scalar`, `map` fields is available.
+            def init_with(coder)
+              @data = {
+                tag: coder.tag,       # This is the custom YAML tag, like !reference or !flatten
+                style: coder.style,
+                seq: coder.seq,       # This holds Array data
+                scalar: coder.scalar, # This holds data of basic types, like String.
+                map: coder.map        # This holds Hash data.
+              }
+            end
+
+            def valid?
+              raise NotImplementedError
+            end
+
+            def resolve(resolver)
+              raise NotValidError, validation_error_message unless valid?
+              raise CircularReferenceError, circular_error_message if resolving?
+              return resolved_value if resolved?
+
+              self.resolved_status = :in_progress
+              self.resolved_value = _resolve(resolver)
+              self.resolved_status = :done
+              resolved_value
+            end
+
+            private
+
+            def _resolve(resolver)
+              raise NotImplementedError
+            end
+
+            def resolved?
+              resolved_status == :done
+            end
+
+            def resolving?
+              resolved_status == :in_progress
+            end
+
+            def circular_error_message
+              "#{data[:tag]} #{data[:seq].inspect} is part of a circular chain"
+            end
+
+            def validation_error_message
+              "#{data[:tag]} #{(data[:scalar].presence || data[:map].presence || data[:seq]).inspect} is not valid"
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/yaml/tags/reference.rb b/lib/gitlab/ci/config/yaml/tags/reference.rb
new file mode 100644
index 00000000000..22822614b67
--- /dev/null
+++ b/lib/gitlab/ci/config/yaml/tags/reference.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Yaml
+        module Tags
+          class Reference < Base
+            MissingReferenceError = Class.new(Tags::TagError)
+
+            def self.tag
+              '!reference'
+            end
+
+            override :valid?
+            def valid?
+              data[:seq].is_a?(Array) &&
+                !data[:seq].empty? &&
+                data[:seq].all? { |identifier| identifier.is_a?(String) }
+            end
+
+            private
+
+            def location
+              data[:seq].to_a.map(&:to_sym)
+            end
+
+            override :_resolve
+            def _resolve(resolver)
+              object = resolver.config.dig(*location)
+              value = resolver.deep_resolve(object)
+
+              raise MissingReferenceError, missing_ref_error_message unless value
+
+              value
+            end
+
+            def missing_ref_error_message
+              "#{data[:tag]} #{data[:seq].inspect} could not be found"
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/yaml/tags/resolver.rb b/lib/gitlab/ci/config/yaml/tags/resolver.rb
new file mode 100644
index 00000000000..e207ec296b6
--- /dev/null
+++ b/lib/gitlab/ci/config/yaml/tags/resolver.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Yaml
+        module Tags
+          # This class is the entry point for transforming custom YAML tags back
+          # into primitive objects.
+          # Usage: `Resolver.new(a_hash_including_custom_tag_objects).to_hash`
+          #
+          class Resolver
+            attr_reader :config
+
+            def initialize(config)
+              @config = config.deep_dup
+            end
+
+            def to_hash
+              deep_resolve(config)
+            end
+
+            def deep_resolve(object)
+              case object
+              when Array
+                object.map(&method(:resolve_wrapper))
+              when Hash
+                object.deep_transform_values(&method(:resolve_wrapper))
+              else
+                resolve_wrapper(object)
+              end
+            end
+
+            def resolve_wrapper(object)
+              if object.respond_to?(:resolve)
+                object.resolve(self)
+              else
+                object
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end