Allow to recursively expand includesallow-to-recursively-include

7 files changed, 67 insertions, 11 deletions

diff --git a/changelogs/unreleased/allow-to-recursively-include.yml b/changelogs/unreleased/allow-to-recursively-include.yml
new file mode 100644
index 00000000000..116ed8c6562
--- /dev/null
+++ b/changelogs/unreleased/allow-to-recursively-include.yml
@@ -0,0 +1,5 @@
+---
+title: Allow to recursively expand includes
+merge_request:
+author:
+type: added
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb
index a747886093c..827d9d63efa 100644
--- a/lib/gitlab/ci/config/external/file/base.rb
+++ b/lib/gitlab/ci/config/external/file/base.rb
@@ -12,7 +12,7 @@ module Gitlab
 
             YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze
 
-            Context = Struct.new(:project, :sha, :user)
+            Context = Struct.new(:project, :sha, :user, :depth)
 
             def initialize(params, context)
               @params = params
@@ -43,13 +43,27 @@ module Gitlab
             end
 
             def to_hash
-              @hash ||= Gitlab::Config::Loader::Yaml.new(content).load!
-            rescue Gitlab::Config::Loader::FormatError
-              nil
+              expanded_yaml
             end
 
             protected
 
+            def expanded_yaml
+              return unless content_yaml
+
+              strong_memoize(:expanded_content_yaml) do
+                expand_includes(content_yaml)
+              end
+            end
+
+            def content_yaml
+              strong_memoize(:content_yaml) do
+                Gitlab::Config::Loader::Yaml.new(content).load!
+              end
+            rescue Gitlab::Config::Loader::FormatError
+              nil
+            end
+
             def validate!
               validate_location!
               validate_content! if errors.none?
@@ -73,6 +87,14 @@ module Gitlab
                 errors.push("Included file `#{location}` does not have valid YAML syntax!")
               end
             end
+
+            def expand_includes(hash)
+              External::Processor.new(hash, **expand_context).perform
+            end
+
+            def expand_context
+              { project: nil, sha: nil, user: nil, depth: context.depth.to_i + 1 }
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb
index 2535d178ba8..229a06451e8 100644
--- a/lib/gitlab/ci/config/external/file/local.rb
+++ b/lib/gitlab/ci/config/external/file/local.rb
@@ -31,6 +31,13 @@ module Gitlab
             def fetch_local_content
               context.project.repository.blob_data_at(context.sha, location)
             end
+
+            def expand_context
+              super.merge(
+                project: context.project,
+                sha: context.sha,
+                user: context.user)
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb
index e75540dbe5a..b828f77835c 100644
--- a/lib/gitlab/ci/config/external/file/project.rb
+++ b/lib/gitlab/ci/config/external/file/project.rb
@@ -64,6 +64,13 @@ module Gitlab
                 project.commit(ref_name).try(:sha)
               end
             end
+
+            def expand_context
+              super.merge(
+                project: project,
+                sha: sha,
+                user: context.user)
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb
index 54f4cf74c4d..d5ebe613b54 100644
--- a/lib/gitlab/ci/config/external/file/template.rb
+++ b/lib/gitlab/ci/config/external/file/template.rb
@@ -43,6 +43,10 @@ module Gitlab
             def fetch_template_content
               Gitlab::Template::GitlabCiYmlTemplate.find(template_name, project)&.content
             end
+
+            def expand_context
+              super.merge(user: context.user)
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index 108bfd5eb43..9406d827309 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -7,6 +7,8 @@ module Gitlab
         class Mapper
           include Gitlab::Utils::StrongMemoize
 
+          MAX_DEPTH = 3
+
           FILE_CLASSES = [
             External::File::Remote,
             External::File::Template,
@@ -14,16 +16,25 @@ module Gitlab
             External::File::Project
           ].freeze
 
-          AmbigiousSpecificationError = Class.new(StandardError)
+          Error = Class.new(StandardError)
+          AmbigiousSpecificationError = Class.new(Error)
+          TooManyIncludes = Class.new(Error)
 
-          def initialize(values, project:, sha:, user:)
+          def initialize(values, project:, sha:, user:, depth: 0)
             @locations = Array.wrap(values.fetch(:include, []))
             @project = project
             @sha = sha
             @user = user
+            @depth = depth.to_i
           end
 
           def process
+            return [] if locations.empty?
+
+            if @depth >= MAX_DEPTH
+              raise TooManyIncludes, "You can nest at most #{MAX_DEPTH} includes"
+            end
+
             locations
               .compact
               .map(&method(:normalize_location))
@@ -32,7 +43,7 @@ module Gitlab
 
           private
 
-          attr_reader :locations, :project, :sha, :user
+          attr_reader :locations, :project, :sha, :user, :depth
 
           # convert location if String to canonical form
           def normalize_location(location)
@@ -63,7 +74,7 @@ module Gitlab
 
           def context
             strong_memoize(:context) do
-              External::File::Base::Context.new(project, sha, user)
+              External::File::Base::Context.new(project, sha, user, depth)
             end
           end
         end
diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb
index 69bc164a039..ca72ba92da5 100644
--- a/lib/gitlab/ci/config/external/processor.rb
+++ b/lib/gitlab/ci/config/external/processor.rb
@@ -7,11 +7,11 @@ module Gitlab
         class Processor
           IncludeError = Class.new(StandardError)
 
-          def initialize(values, project:, sha:, user:)
+          def initialize(values, project:, sha:, user:, depth: 0)
             @values = values
-            @external_files = External::Mapper.new(values, project: project, sha: sha, user: user).process
+            @external_files = External::Mapper.new(values, project: project, sha: sha, user: user, depth: depth).process
             @content = {}
-          rescue External::Mapper::AmbigiousSpecificationError => e
+          rescue External::Mapper::Error => e
             raise IncludeError, e.message
           end