Merge branch '42861-move-include-external-files-in-gitlab-ci-yml-from-starter-to-libre' into 'master'

Resolve "Move "include external files in .gitlab-ci.yml" from Starter to Libre"

Closes #42861

See merge request gitlab-org/gitlab-ce!21603

6 files changed, 195 insertions, 3 deletions

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 46dad59eb8c..fe98d25af29 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -1,6 +1,6 @@
 module Gitlab
   module Ci
-    ##
+    #
     # Base GitLab CI Configuration facade
     #
     class Config
@@ -15,6 +15,8 @@ module Gitlab
         @global.compose!
       rescue Loader::FormatError, Extendable::ExtensionError => e
         raise Config::ConfigError, e.message
+      rescue ::Gitlab::Ci::External::Processor::FileError => e
+        raise ::Gitlab::Ci::YamlProcessor::ValidationError, e.message
       end
 
       def valid?
@@ -64,9 +66,22 @@ module Gitlab
         @global.jobs_value
       end
 
-      # 'opts' argument is used in EE see /ee/lib/ee/gitlab/ci/config.rb
+      private
+
       def build_config(config, opts = {})
-        Loader.new(config).load!
+        initial_config = Loader.new(config).load!
+        project = opts.fetch(:project, nil)
+
+        if project
+          process_external_files(initial_config, project, opts)
+        else
+          initial_config
+        end
+      end
+
+      def process_external_files(config, project, opts)
+        sha = opts.fetch(:sha) { project.repository.root_ref_sha }
+        ::Gitlab::Ci::External::Processor.new(config, project, sha).perform
       end
     end
   end
diff --git a/lib/gitlab/ci/external/file/base.rb b/lib/gitlab/ci/external/file/base.rb
new file mode 100644
index 00000000000..f4da07b0b02
--- /dev/null
+++ b/lib/gitlab/ci/external/file/base.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module External
+      module File
+        class Base
+          YAML_WHITELIST_EXTENSION = /(yml|yaml)$/i.freeze
+
+          def initialize(location, opts = {})
+            @location = location
+          end
+
+          def valid?
+            location.match(YAML_WHITELIST_EXTENSION) && content
+          end
+
+          def content
+            raise NotImplementedError, 'content must be implemented and return a string or nil'
+          end
+
+          def error_message
+            raise NotImplementedError, 'error_message must be implemented and return a string'
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external/file/local.rb b/lib/gitlab/ci/external/file/local.rb
new file mode 100644
index 00000000000..1aa7f687507
--- /dev/null
+++ b/lib/gitlab/ci/external/file/local.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module External
+      module File
+        class Local < Base
+          attr_reader :location, :project, :sha
+
+          def initialize(location, opts = {})
+            super
+
+            @project = opts.fetch(:project)
+            @sha = opts.fetch(:sha)
+          end
+
+          def content
+            @content ||= fetch_local_content
+          end
+
+          def error_message
+            "Local file '#{location}' is not valid."
+          end
+
+          private
+
+          def fetch_local_content
+            project.repository.blob_data_at(sha, location)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external/file/remote.rb b/lib/gitlab/ci/external/file/remote.rb
new file mode 100644
index 00000000000..59bb3e8999e
--- /dev/null
+++ b/lib/gitlab/ci/external/file/remote.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module External
+      module File
+        class Remote < Base
+          include Gitlab::Utils::StrongMemoize
+          attr_reader :location
+
+          def content
+            return @content if defined?(@content)
+
+            @content = strong_memoize(:content) do
+              begin
+                Gitlab::HTTP.get(location)
+              rescue Gitlab::HTTP::Error, Timeout::Error, SocketError, Gitlab::HTTP::BlockedUrlError
+                nil
+              end
+            end
+          end
+
+          def error_message
+            "Remote file '#{location}' is not valid."
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external/mapper.rb b/lib/gitlab/ci/external/mapper.rb
new file mode 100644
index 00000000000..58bd6a19acf
--- /dev/null
+++ b/lib/gitlab/ci/external/mapper.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module External
+      class Mapper
+        def initialize(values, project, sha)
+          @locations = Array(values.fetch(:include, []))
+          @project = project
+          @sha = sha
+        end
+
+        def process
+          locations.map { |location| build_external_file(location) }
+        end
+
+        private
+
+        attr_reader :locations, :project, :sha
+
+        def build_external_file(location)
+          if ::Gitlab::UrlSanitizer.valid?(location)
+            Gitlab::Ci::External::File::Remote.new(location)
+          else
+            options = { project: project, sha: sha }
+            Gitlab::Ci::External::File::Local.new(location, options)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external/processor.rb b/lib/gitlab/ci/external/processor.rb
new file mode 100644
index 00000000000..76cf3ce89f9
--- /dev/null
+++ b/lib/gitlab/ci/external/processor.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module External
+      class Processor
+        FileError = Class.new(StandardError)
+
+        def initialize(values, project, sha)
+          @values = values
+          @external_files = Gitlab::Ci::External::Mapper.new(values, project, sha).process
+          @content = {}
+        end
+
+        def perform
+          return values if external_files.empty?
+
+          external_files.each do |external_file|
+            validate_external_file(external_file)
+            @content.deep_merge!(content_of(external_file))
+          end
+
+          append_inline_content
+          remove_include_keyword
+        end
+
+        private
+
+        attr_reader :values, :external_files, :content
+
+        def validate_external_file(external_file)
+          unless external_file.valid?
+            raise FileError, external_file.error_message
+          end
+        end
+
+        def content_of(external_file)
+          Gitlab::Ci::Config::Loader.new(external_file.content).load!
+        end
+
+        def append_inline_content
+          @content.deep_merge!(@values)
+        end
+
+        def remove_include_keyword
+          content.delete(:include)
+          content
+        end
+      end
+    end
+  end
+end