Add latest changes from gitlab-org/gitlab@13-10-stable-eev13.10.0-rc40

4 files changed, 144 insertions, 87 deletions

diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb
index f7bbb58df7e..e2a8af9c26b 100644
--- a/lib/gitlab/ci/variables/collection.rb
+++ b/lib/gitlab/ci/variables/collection.rb
@@ -6,14 +6,22 @@ module Gitlab
       class Collection
         include Enumerable
 
-        def initialize(variables = [])
+        attr_reader :errors
+
+        def initialize(variables = [], errors = nil)
           @variables = []
+          @variables_by_key = {}
+          @errors = errors
 
           variables.each { |variable| self.append(variable) }
         end
 
         def append(resource)
-          tap { @variables.append(Collection::Item.fabricate(resource)) }
+          item = Collection::Item.fabricate(resource)
+          @variables.append(item)
+          @variables_by_key[item[:key]] = item
+
+          self
         end
 
         def concat(resources)
@@ -33,15 +41,67 @@ module Gitlab
           end
         end
 
+        def [](key)
+          @variables_by_key[key]
+        end
+
+        def size
+          @variables.size
+        end
+
         def to_runner_variables
           self.map(&:to_runner_variable)
         end
 
         def to_hash
           self.to_runner_variables
-            .map { |env| [env.fetch(:key), env.fetch(:value)] }
-            .to_h.with_indifferent_access
+            .to_h { |env| [env.fetch(:key), env.fetch(:value)] }
+            .with_indifferent_access
         end
+
+        def reject(&block)
+          Collection.new(@variables.reject(&block))
+        end
+
+        def expand_value(value, keep_undefined: false)
+          value.gsub(ExpandVariables::VARIABLES_REGEXP) do
+            match = Regexp.last_match
+            result = @variables_by_key[match[1] || match[2]]&.value
+            result ||= match[0] if keep_undefined
+            result
+          end
+        end
+
+        def sort_and_expand_all(project, keep_undefined: false)
+          return self if Feature.disabled?(:variable_inside_variable, project)
+
+          sorted = Sort.new(self)
+          return self.class.new(self, sorted.errors) unless sorted.valid?
+
+          new_collection = self.class.new
+
+          sorted.tsort.each do |item|
+            unless item.depends_on
+              new_collection.append(item)
+              next
+            end
+
+            # expand variables as they are added
+            variable = item.to_runner_variable
+            variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined)
+            new_collection.append(variable)
+          end
+
+          new_collection
+        end
+
+        def to_s
+          "#{@variables_by_key.keys}, @errors='#{@errors}'"
+        end
+
+        protected
+
+        attr_reader :variables
       end
     end
   end
diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb
index 84a9280e507..77da2c4cb91 100644
--- a/lib/gitlab/ci/variables/collection/item.rb
+++ b/lib/gitlab/ci/variables/collection/item.rb
@@ -5,13 +5,21 @@ module Gitlab
     module Variables
       class Collection
         class Item
-          def initialize(key:, value:, public: true, file: false, masked: false)
+          include Gitlab::Utils::StrongMemoize
+
+          def initialize(key:, value:, public: true, file: false, masked: false, raw: false)
             raise ArgumentError, "`#{key}` must be of type String or nil value, while it was: #{value.class}" unless
               value.is_a?(String) || value.nil?
 
-            @variable = {
-              key: key, value: value, public: public, file: file, masked: masked
-            }
+            @variable = { key: key, value: value, public: public, file: file, masked: masked, raw: raw }
+          end
+
+          def value
+            @variable.fetch(:value)
+          end
+
+          def raw
+            @variable.fetch(:raw)
           end
 
           def [](key)
@@ -22,6 +30,16 @@ module Gitlab
             to_runner_variable == self.class.fabricate(other).to_runner_variable
           end
 
+          def depends_on
+            strong_memoize(:depends_on) do
+              next if raw
+
+              next unless ExpandVariables.possible_var_reference?(value)
+
+              value.scan(ExpandVariables::VARIABLES_REGEXP).map(&:first)
+            end
+          end
+
           ##
           # If `file: true` has been provided we expose it, otherwise we
           # don't expose `file` attribute at all (stems from what the runner
@@ -29,7 +47,7 @@ module Gitlab
           #
           def to_runner_variable
             @variable.reject do |hash_key, hash_value|
-              hash_key == :file && hash_value == false
+              (hash_key == :file || hash_key == :raw) && hash_value == false
             end
           end
 
@@ -45,6 +63,12 @@ module Gitlab
               raise ArgumentError, "Unknown `#{resource.class}` variable resource!"
             end
           end
+
+          def to_s
+            return to_runner_variable.to_s unless depends_on
+
+            "#{to_runner_variable}, depends_on=#{depends_on}"
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/variables/collection/sort.rb b/lib/gitlab/ci/variables/collection/sort.rb
new file mode 100644
index 00000000000..90a929b8a07
--- /dev/null
+++ b/lib/gitlab/ci/variables/collection/sort.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Variables
+      class Collection
+        class Sort
+          include TSort
+          include Gitlab::Utils::StrongMemoize
+
+          def initialize(collection)
+            raise(ArgumentError, "A Gitlab::Ci::Variables::Collection object was expected") unless
+              collection.is_a?(Collection)
+
+            @collection = collection
+          end
+
+          def valid?
+            errors.nil?
+          end
+
+          # errors sorts an array of variables, ignoring unknown variable references,
+          # and returning an error string if a circular variable reference is found
+          def errors
+            strong_memoize(:errors) do
+              # Check for cyclic dependencies and build error message in that case
+              cyclic_vars = each_strongly_connected_component.filter_map do |component|
+                component.map { |v| v[:key] }.inspect if component.size > 1
+              end
+
+              "circular variable reference detected: #{cyclic_vars.join(', ')}" if cyclic_vars.any?
+            end
+          end
+
+          private
+
+          def tsort_each_node(&block)
+            @collection.each(&block)
+          end
+
+          def tsort_each_child(var_item, &block)
+            depends_on = var_item.depends_on
+            return unless depends_on
+
+            depends_on.filter_map { |var_ref_name| @collection[var_ref_name] }.each(&block)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/variables/collection/sorted.rb b/lib/gitlab/ci/variables/collection/sorted.rb
deleted file mode 100644
index e641df10462..00000000000
--- a/lib/gitlab/ci/variables/collection/sorted.rb
+++ /dev/null
@@ -1,78 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Ci
-    module Variables
-      class Collection
-        class Sorted
-          include TSort
-          include Gitlab::Utils::StrongMemoize
-
-          def initialize(variables, project)
-            @variables = variables
-            @project = project
-          end
-
-          def valid?
-            errors.nil?
-          end
-
-          # errors sorts an array of variables, ignoring unknown variable references,
-          # and returning an error string if a circular variable reference is found
-          def errors
-            return if Feature.disabled?(:variable_inside_variable, @project)
-
-            strong_memoize(:errors) do
-              # Check for cyclic dependencies and build error message in that case
-              errors = each_strongly_connected_component.filter_map do |component|
-                component.map { |v| v[:key] }.inspect if component.size > 1
-              end
-
-              "circular variable reference detected: #{errors.join(', ')}" if errors.any?
-            end
-          end
-
-          # sort sorts an array of variables, ignoring unknown variable references.
-          # If a circular variable reference is found, the original array is returned
-          def sort
-            return @variables if Feature.disabled?(:variable_inside_variable, @project)
-            return @variables if errors
-
-            tsort
-          end
-
-          private
-
-          def tsort_each_node(&block)
-            @variables.each(&block)
-          end
-
-          def tsort_each_child(variable, &block)
-            each_variable_reference(variable[:value], &block)
-          end
-
-          def input_vars
-            strong_memoize(:input_vars) do
-              @variables.index_by { |env| env.fetch(:key) }
-            end
-          end
-
-          def walk_references(value)
-            return unless ExpandVariables.possible_var_reference?(value)
-
-            value.scan(ExpandVariables::VARIABLES_REGEXP) do |var_ref|
-              yield(input_vars, var_ref.first)
-            end
-          end
-
-          def each_variable_reference(value)
-            walk_references(value) do |vars_hash, ref_var_name|
-              variable = vars_hash.dig(ref_var_name)
-              yield variable if variable
-            end
-          end
-        end
-      end
-    end
-  end
-end