Improve external architecture

CE mirror of 4f17c7b2c30188302e6a73421acbf5a09fb2c823

9 files changed, 155 insertions, 125 deletions

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 4a0d67720a9..1779fbb2a39 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -6,7 +6,7 @@ module Gitlab
     class Config
       ConfigError = Class.new(StandardError)
 
-      def initialize(config, project = nil, opts = {})
+      def initialize(config, opts = {})
         @config = Config::Extendable
           .new(build_config(config, opts))
           .to_hash
@@ -64,17 +64,21 @@ module Gitlab
         @global.jobs_value
       end
 
-      # 'opts' argument is used in EE see /ee/lib/ee/gitlab/ci/config.rb
-      def build_config(config, project,  opts = {})
+      def build_config(config, opts = {})
         initial_config = Loader.new(config).load!
+        project = opts.fetch(:project, nil)
 
-        if project.present?
-          processor = ::Gitlab::Ci::ExternalFiles::Processor.new(initial_config, project)
-          processor.perform
+        if project
+          process_external_files(initial_config, project, opts)
         else
           initial_config
         end
       end
+
+      def process_external_files(config, project, opts)
+        branch_name = opts.fetch(:branch_name, project.default_branch)
+        ::Gitlab::Ci::External::Processor.new(config, project, branch_name).perform
+      end
     end
   end
 end
diff --git a/lib/gitlab/ci/external/file/local.rb b/lib/gitlab/ci/external/file/local.rb
new file mode 100644
index 00000000000..dc1ba923411
--- /dev/null
+++ b/lib/gitlab/ci/external/file/local.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module Ci
+    module External
+      module File
+        class Local
+          attr_reader :value, :project, :branch_name
+
+          def initialize(value, project, branch_name)
+            @value = value
+            @project = project
+            @branch_name = branch_name
+          end
+
+          def valid?
+            commit && local_file_content
+          end
+
+          def content
+            local_file_content
+          end
+
+          private
+
+          def commit
+            @commit ||= project.repository.commit(branch_name)
+          end
+
+          def local_file_content
+            @local_file_content ||= project.repository.blob_data_at(commit.sha, value)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external/file/remote.rb b/lib/gitlab/ci/external/file/remote.rb
new file mode 100644
index 00000000000..7d8cd1957ad
--- /dev/null
+++ b/lib/gitlab/ci/external/file/remote.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module Ci
+    module External
+      module File
+        class Remote
+          attr_reader :value
+
+          def initialize(value)
+            @value = value
+          end
+
+          def valid?
+            ::Gitlab::UrlSanitizer.valid?(value) && content
+          end
+
+          def content
+            HTTParty.get(value)
+          rescue HTTParty::Error, Timeout::Error
+            false
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external/mapper.rb b/lib/gitlab/ci/external/mapper.rb
new file mode 100644
index 00000000000..f2e5ec972df
--- /dev/null
+++ b/lib/gitlab/ci/external/mapper.rb
@@ -0,0 +1,31 @@
+module Gitlab
+  module Ci
+    module External
+      class Mapper
+        def initialize(values, project, branch_name)
+          @paths = Array(values.fetch(:include, []))
+          @project = project
+          @branch_name = branch_name
+        end
+
+        def process
+          paths.map { |path| build_external_file(path) }
+        end
+
+        private
+
+        attr_reader :paths, :project, :branch_name
+
+        def build_external_file(path)
+          remote_file = Gitlab::Ci::External::File::Remote.new(path)
+
+          if remote_file.valid?
+            remote_file
+          else
+            ::Gitlab::Ci::External::File::Local.new(path, project, branch_name)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external/processor.rb b/lib/gitlab/ci/external/processor.rb
new file mode 100644
index 00000000000..5cae0166346
--- /dev/null
+++ b/lib/gitlab/ci/external/processor.rb
@@ -0,0 +1,50 @@
+module Gitlab
+  module Ci
+    module External
+      class Processor
+        FileError = Class.new(StandardError)
+
+        def initialize(values, project, branch_name)
+          @values = values
+          @external_files = ::Gitlab::Ci::External::Mapper.new(values, project, branch_name).process
+          @content = {}
+        end
+
+        def perform
+          return values if external_files.empty?
+
+          external_files.each do |external_file|
+            validate_external_file(external_file)
+            @content.merge!(content_of(external_file))
+          end
+
+          append_external_content
+          remove_include_keyword
+        end
+
+        private
+
+        attr_reader :values, :external_files, :content
+
+        def validate_external_file(external_file)
+          unless external_file.valid?
+            raise FileError, "External file: '#{external_file.value}' should be a valid local or remote file"
+          end
+        end
+
+        def content_of(external_file)
+          ::Gitlab::Ci::Config::Loader.new(external_file.content).load!
+        end
+
+        def append_external_content
+          @content.merge!(@values)
+        end
+
+        def remove_include_keyword
+          content.delete(:include)
+          content
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external_files/external_file.rb b/lib/gitlab/ci/external_files/external_file.rb
deleted file mode 100644
index 4fffdc5f716..00000000000
--- a/lib/gitlab/ci/external_files/external_file.rb
+++ /dev/null
@@ -1,42 +0,0 @@
-module Gitlab
-  module Ci
-    module ExternalFiles
-      class ExternalFile
-        attr_reader :value, :project
-
-        def initialize(value, project)
-          @value = value
-          @project = project
-        end
-
-        def content
-          if remote_url?
-            HTTParty.get(value)
-          else
-            local_file_content
-          end
-        rescue HTTParty::Error, Timeout::Error
-          nil
-        end
-
-        def valid?
-          remote_url? || local_file_content
-        end
-
-        private
-
-        def remote_url?
-          ::Gitlab::UrlSanitizer.valid?(value)
-        end
-
-        def local_file_content
-          project.repository.blob_data_at(sha, value)
-        end
-
-        def sha
-          @sha ||= project.repository.commit.sha
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/external_files/mapper.rb b/lib/gitlab/ci/external_files/mapper.rb
deleted file mode 100644
index fa252cc3b9e..00000000000
--- a/lib/gitlab/ci/external_files/mapper.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-module Gitlab
-  module Ci
-    module ExternalFiles
-      class Mapper
-        def initialize(values, project)
-          @paths = values.fetch(:include, [])
-          @project = project
-        end
-
-        def process
-          if paths.is_a?(String)
-            [build_external_file(paths)]
-          else
-            paths.map { |path| build_external_file(path) }
-          end
-        end
-
-        private
-
-        attr_reader :paths, :project
-
-        def build_external_file(path)
-          ::Gitlab::Ci::ExternalFiles::ExternalFile.new(path, project)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/external_files/processor.rb b/lib/gitlab/ci/external_files/processor.rb
deleted file mode 100644
index 5657f025084..00000000000
--- a/lib/gitlab/ci/external_files/processor.rb
+++ /dev/null
@@ -1,45 +0,0 @@
-module Gitlab
-  module Ci
-    module ExternalFiles
-      class Processor
-        ExternalFileError = Class.new(StandardError)
-
-        def initialize(values, project)
-          @values = values
-          @external_files = ::Gitlab::Ci::ExternalFiles::Mapper.new(values, project).process
-        end
-
-        def perform
-          return values if external_files.empty?
-
-          external_files.each do |external_file|
-            validate_external_file(external_file)
-            append_external_content(external_file)
-          end
-
-          remove_include_keyword
-        end
-
-        private
-
-        attr_reader :values, :external_files
-
-        def validate_external_file(external_file)
-          unless external_file.valid?
-            raise ExternalFileError, "External file: '#{external_file.value}' should be a valid local or remote file"
-          end
-        end
-
-        def append_external_content(external_file)
-          external_values = ::Gitlab::Ci::Config::Loader.new(external_file.content).load!
-          @values.merge!(external_values)
-        end
-
-        def remove_include_keyword
-          values.delete(:include)
-          values
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 0f79fbede9f..67f5d2d6ae4 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -7,8 +7,8 @@ module Gitlab
 
       attr_reader :cache, :stages, :jobs
 
-      def initialize(config, project = nil, opts = {})
-        @ci_config = Gitlab::Ci::Config.new(config, project, opts)
+      def initialize(config, opts = {})
+        @ci_config = Gitlab::Ci::Config.new(config, opts)
         @config = @ci_config.to_hash
 
         unless @ci_config.valid?
@@ -73,11 +73,11 @@ module Gitlab
         end
       end
 
-      def self.validation_message(content, project = nil, opts = {})
+      def self.validation_message(content, opts = {})
         return 'Please provide content of .gitlab-ci.yml' if content.blank?
 
         begin
-          Gitlab::Ci::YamlProcessor.new(content, project, opts)
+          Gitlab::Ci::YamlProcessor.new(content, opts)
           nil
         rescue ValidationError, ::Gitlab::Ci::ExternalFiles::Processor::ExternalFileError => e
           e.message