From c3e33f06c2920a9f032ee8166cccf8423bd24b78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Fri, 7 Sep 2018 19:57:57 +0200
Subject: Build barebones for ExternalFiles libraries

CE mirror of 30ca00f17034f654403ec7cb5dc370d1e5db8152
---
 lib/gitlab/ci/config/entry/global.rb          | 10 +++++-
 lib/gitlab/ci/config/entry/includes.rb        | 22 +++++++++++++
 lib/gitlab/ci/config/entry/validators.rb      | 32 +++++++++++++++++++
 lib/gitlab/ci/external_files/external_file.rb | 38 ++++++++++++++++++++++
 lib/gitlab/ci/external_files/mapper.rb        | 27 ++++++++++++++++
 lib/gitlab/ci/external_files/processor.rb     | 46 +++++++++++++++++++++++++++
 6 files changed, 174 insertions(+), 1 deletion(-)
 create mode 100644 lib/gitlab/ci/config/entry/includes.rb
 create mode 100644 lib/gitlab/ci/external_files/external_file.rb
 create mode 100644 lib/gitlab/ci/external_files/mapper.rb
 create mode 100644 lib/gitlab/ci/external_files/processor.rb

(limited to 'lib')

diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb
index a4ec8f0ff2f..401e566de77 100644
--- a/lib/gitlab/ci/config/entry/global.rb
+++ b/lib/gitlab/ci/config/entry/global.rb
@@ -33,11 +33,15 @@ module Gitlab
           entry :cache, Entry::Cache,
             description: 'Configure caching between build jobs.'
 
+          entry :includes, Entry::Includes,
+            description: 'External GitlLab Ci files'
+
           helpers :before_script, :image, :services, :after_script,
-                  :variables, :stages, :types, :cache, :jobs
+                  :variables, :stages, :types, :cache, :jobs, :includes
 
           def compose!(_deps = nil)
             super(self) do
+              append_external_files!
               compose_jobs!
               compose_deprecated_entries!
             end
@@ -45,6 +49,10 @@ module Gitlab
 
           private
 
+          def append_external_files!
+            return if includes_value.nil?
+          end
+
           def compose_jobs!
             factory = Entry::Factory.new(Entry::Jobs)
               .value(@config.except(*self.class.nodes.keys))
diff --git a/lib/gitlab/ci/config/entry/includes.rb b/lib/gitlab/ci/config/entry/includes.rb
new file mode 100644
index 00000000000..c7a8c7960e8
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/includes.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a Docker image.
+        #
+        class Includes < Node
+          include Validatable
+
+          validations do
+            validates :config, array_or_string: true, external_file: true, allow_nil: true
+          end
+
+          def value
+            Array(@config)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb
index b3c889ee92f..5b4d5d53821 100644
--- a/lib/gitlab/ci/config/entry/validators.rb
+++ b/lib/gitlab/ci/config/entry/validators.rb
@@ -60,6 +60,38 @@ module Gitlab
             end
           end
 
+          class ArrayOrStringValidator < ActiveModel::EachValidator
+            def validate_each(record, attribute, value)
+              unless value.is_a?(Array) || value.is_a?(String)
+                record.errors.add(attribute, 'should be an array or a string')
+              end
+            end
+          end
+
+          class ExternalFileValidator < ActiveModel::EachValidator
+            def validate_each(record, attribute, value)
+              if value.is_a?(Array)
+                value.each do |path|
+                  validate_external_file(path, record, attribute)
+                end
+              else
+                validate_external_file(value, record, attribute)
+              end
+            end
+
+            private
+
+            def validate_external_file(value, record, attribute)
+              unless valid_url?(value)
+                record.errors.add(attribute, 'should be a valid local or remote file')
+              end
+            end
+
+            def valid_url?(value)
+              Gitlab::UrlSanitizer.valid?(value) || File.exists?("#{Rails.root}/#{value}")
+            end
+          end
+
           class KeyValidator < ActiveModel::EachValidator
             include LegacyValidationHelpers
 
diff --git a/lib/gitlab/ci/external_files/external_file.rb b/lib/gitlab/ci/external_files/external_file.rb
new file mode 100644
index 00000000000..9bad8231c42
--- /dev/null
+++ b/lib/gitlab/ci/external_files/external_file.rb
@@ -0,0 +1,38 @@
+require 'open-uri'
+
+module Gitlab
+  module Ci
+    module ExternalFiles
+      class ExternalFile
+
+        def initialize(value)
+          @value = value
+        end
+
+        def content
+          if remote_url?
+            open(value).read
+          else
+            File.read(base_path)
+          end
+        end
+
+        def valid?
+          remote_url? || File.exists?(base_path) 
+        end
+
+        private
+
+        attr_reader :value
+
+        def base_path
+          "#{Rails.root}/#{value}"
+        end
+
+        def remote_url?
+          ::Gitlab::UrlSanitizer.valid?(value)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external_files/mapper.rb b/lib/gitlab/ci/external_files/mapper.rb
new file mode 100644
index 00000000000..5deb2f5a2e5
--- /dev/null
+++ b/lib/gitlab/ci/external_files/mapper.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Ci
+    module ExternalFiles
+      class Mapper
+
+        def self.fetch_paths(values)
+          paths = values.fetch(:includes, [])
+          normalize_paths(paths)
+        end
+
+        private
+
+        def self.normalize_paths(paths)
+          if paths.is_a?(String)
+            [build_external_file(paths)]
+          else
+            paths.map { |path| build_external_file(path) }
+          end
+        end
+
+        def self.build_external_file(path)
+          ::Gitlab::Ci::ExternalFiles::ExternalFile.new(path)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external_files/processor.rb b/lib/gitlab/ci/external_files/processor.rb
new file mode 100644
index 00000000000..f38bc7633b5
--- /dev/null
+++ b/lib/gitlab/ci/external_files/processor.rb
@@ -0,0 +1,46 @@
+module Gitlab
+  module Ci
+    module ExternalFiles
+      class Processor
+        ExternalFileError = Class.new(StandardError)
+
+        def initialize(values)
+          @values = values
+          @external_files = ::Gitlab::Ci::ExternalFiles::Mapper.fetch_paths(values)
+        end
+
+        def perform
+          return values if external_files.empty?
+
+          external_files.each do |external_file|
+            validate_external_file(external_file)
+            append_external_content(external_file)
+          end
+
+          remove_include_keyword
+        end
+
+        private
+
+        attr_reader :values, :external_files
+
+        def validate_external_file(external_file)
+          unless external_file.valid? 
+            raise ExternalFileError, 'External files should be a valid local or remote file'
+          end
+        end
+
+        def append_external_content(external_file)
+          external_values = ::Gitlab::Ci::Config::Loader.new(external_file.content).load!
+          @values.merge!(external_values)
+        end
+
+        def remove_include_keyword
+          values.delete(:includes)
+          values
+        end
+
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From 5f33690ebbe19dd8698bf8b432a5e6740305f007 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Fri, 7 Sep 2018 20:53:28 +0200
Subject: Load external files in config

CE mirror of 8e03a6619be44fdaf19a6c13284ea8e51377b311
---
 lib/gitlab/ci/config.rb                       |  6 +++--
 lib/gitlab/ci/config/entry/global.rb          | 10 +--------
 lib/gitlab/ci/config/entry/includes.rb        | 22 ------------------
 lib/gitlab/ci/config/entry/validators.rb      | 32 ---------------------------
 lib/gitlab/ci/external_files/external_file.rb |  3 +--
 lib/gitlab/ci/external_files/mapper.rb        |  3 ---
 lib/gitlab/ci/external_files/processor.rb     |  3 +--
 7 files changed, 7 insertions(+), 72 deletions(-)
 delete mode 100644 lib/gitlab/ci/config/entry/includes.rb

(limited to 'lib')

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 46dad59eb8c..94b2fbc082b 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -1,15 +1,17 @@
 module Gitlab
   module Ci
-    ##
+    #
     # Base GitLab CI Configuration facade
     #
     class Config
       ConfigError = Class.new(StandardError)
 
       def initialize(config, opts = {})
-        @config = Config::Extendable
+        initial_config = Config::Extendable
           .new(build_config(config, opts))
           .to_hash
+        processor = ::Gitlab::Ci::ExternalFiles::Processor.new(initial_config)
+        @config = processor.perform
 
         @global = Entry::Global.new(@config)
         @global.compose!
diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb
index 401e566de77..a4ec8f0ff2f 100644
--- a/lib/gitlab/ci/config/entry/global.rb
+++ b/lib/gitlab/ci/config/entry/global.rb
@@ -33,15 +33,11 @@ module Gitlab
           entry :cache, Entry::Cache,
             description: 'Configure caching between build jobs.'
 
-          entry :includes, Entry::Includes,
-            description: 'External GitlLab Ci files'
-
           helpers :before_script, :image, :services, :after_script,
-                  :variables, :stages, :types, :cache, :jobs, :includes
+                  :variables, :stages, :types, :cache, :jobs
 
           def compose!(_deps = nil)
             super(self) do
-              append_external_files!
               compose_jobs!
               compose_deprecated_entries!
             end
@@ -49,10 +45,6 @@ module Gitlab
 
           private
 
-          def append_external_files!
-            return if includes_value.nil?
-          end
-
           def compose_jobs!
             factory = Entry::Factory.new(Entry::Jobs)
               .value(@config.except(*self.class.nodes.keys))
diff --git a/lib/gitlab/ci/config/entry/includes.rb b/lib/gitlab/ci/config/entry/includes.rb
deleted file mode 100644
index c7a8c7960e8..00000000000
--- a/lib/gitlab/ci/config/entry/includes.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        ##
-        # Entry that represents a Docker image.
-        #
-        class Includes < Node
-          include Validatable
-
-          validations do
-            validates :config, array_or_string: true, external_file: true, allow_nil: true
-          end
-
-          def value
-            Array(@config)
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb
index 5b4d5d53821..b3c889ee92f 100644
--- a/lib/gitlab/ci/config/entry/validators.rb
+++ b/lib/gitlab/ci/config/entry/validators.rb
@@ -60,38 +60,6 @@ module Gitlab
             end
           end
 
-          class ArrayOrStringValidator < ActiveModel::EachValidator
-            def validate_each(record, attribute, value)
-              unless value.is_a?(Array) || value.is_a?(String)
-                record.errors.add(attribute, 'should be an array or a string')
-              end
-            end
-          end
-
-          class ExternalFileValidator < ActiveModel::EachValidator
-            def validate_each(record, attribute, value)
-              if value.is_a?(Array)
-                value.each do |path|
-                  validate_external_file(path, record, attribute)
-                end
-              else
-                validate_external_file(value, record, attribute)
-              end
-            end
-
-            private
-
-            def validate_external_file(value, record, attribute)
-              unless valid_url?(value)
-                record.errors.add(attribute, 'should be a valid local or remote file')
-              end
-            end
-
-            def valid_url?(value)
-              Gitlab::UrlSanitizer.valid?(value) || File.exists?("#{Rails.root}/#{value}")
-            end
-          end
-
           class KeyValidator < ActiveModel::EachValidator
             include LegacyValidationHelpers
 
diff --git a/lib/gitlab/ci/external_files/external_file.rb b/lib/gitlab/ci/external_files/external_file.rb
index 9bad8231c42..47d6d1c0fdc 100644
--- a/lib/gitlab/ci/external_files/external_file.rb
+++ b/lib/gitlab/ci/external_files/external_file.rb
@@ -4,7 +4,6 @@ module Gitlab
   module Ci
     module ExternalFiles
       class ExternalFile
-
         def initialize(value)
           @value = value
         end
@@ -18,7 +17,7 @@ module Gitlab
         end
 
         def valid?
-          remote_url? || File.exists?(base_path) 
+          remote_url? || File.exist?(base_path)
         end
 
         private
diff --git a/lib/gitlab/ci/external_files/mapper.rb b/lib/gitlab/ci/external_files/mapper.rb
index 5deb2f5a2e5..4b5d2ddd6a1 100644
--- a/lib/gitlab/ci/external_files/mapper.rb
+++ b/lib/gitlab/ci/external_files/mapper.rb
@@ -2,14 +2,11 @@ module Gitlab
   module Ci
     module ExternalFiles
       class Mapper
-
         def self.fetch_paths(values)
           paths = values.fetch(:includes, [])
           normalize_paths(paths)
         end
 
-        private
-
         def self.normalize_paths(paths)
           if paths.is_a?(String)
             [build_external_file(paths)]
diff --git a/lib/gitlab/ci/external_files/processor.rb b/lib/gitlab/ci/external_files/processor.rb
index f38bc7633b5..221b6f58b98 100644
--- a/lib/gitlab/ci/external_files/processor.rb
+++ b/lib/gitlab/ci/external_files/processor.rb
@@ -25,7 +25,7 @@ module Gitlab
         attr_reader :values, :external_files
 
         def validate_external_file(external_file)
-          unless external_file.valid? 
+          unless external_file.valid?
             raise ExternalFileError, 'External files should be a valid local or remote file'
           end
         end
@@ -39,7 +39,6 @@ module Gitlab
           values.delete(:includes)
           values
         end
-
       end
     end
   end
-- 
cgit v1.2.1


From cd72189fc0127e9993eebaba9569f912394cc5c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Fri, 7 Sep 2018 21:00:52 +0200
Subject: Import external files in Gitlab CI config

CE mirror of a10b777f8cb1902bba3964a4cbad2b1dd6bce785
---
 lib/gitlab/ci/config.rb         | 11 ++++++++---
 lib/gitlab/ci/yaml_processor.rb |  4 ++--
 2 files changed, 10 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 94b2fbc082b..f665ace7c74 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -6,12 +6,17 @@ module Gitlab
     class Config
       ConfigError = Class.new(StandardError)
 
-      def initialize(config, opts = {})
+      def initialize(config, project = nil, opts = {})
         initial_config = Config::Extendable
           .new(build_config(config, opts))
           .to_hash
-        processor = ::Gitlab::Ci::ExternalFiles::Processor.new(initial_config)
-        @config = processor.perform
+
+        if project.present?
+          processor = ::Gitlab::Ci::ExternalFiles::Processor.new(initial_config)
+          @config = processor.perform
+        else
+          @config = initial_config
+        end
 
         @global = Entry::Global.new(@config)
         @global.compose!
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 5d1864ae9e2..702bcd3802d 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -7,8 +7,8 @@ module Gitlab
 
       attr_reader :cache, :stages, :jobs
 
-      def initialize(config, opts = {})
-        @ci_config = Gitlab::Ci::Config.new(config, opts)
+      def initialize(config, project = nil, opts = {})
+        @ci_config = Gitlab::Ci::Config.new(config, project, opts)
         @config = @ci_config.to_hash
 
         unless @ci_config.valid?
-- 
cgit v1.2.1


From 95b296f8ac8578e142efd6a60a582be4da5b09be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Fri, 7 Sep 2018 21:18:26 +0200
Subject: Change ExternalFile to retrieve local file from repository instead of
 GitLab project

CE mirror of 03c6094997023d9c8875ced421a6c9ef39a4af44
---
 lib/gitlab/ci/config.rb                       |  2 +-
 lib/gitlab/ci/external_files/external_file.rb | 21 +++++++++++++--------
 lib/gitlab/ci/external_files/mapper.rb        | 16 ++++++++++------
 lib/gitlab/ci/external_files/processor.rb     |  4 ++--
 lib/gitlab/ci/yaml_processor.rb               |  6 +++---
 5 files changed, 29 insertions(+), 20 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index f665ace7c74..caa3a7c3c86 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -12,7 +12,7 @@ module Gitlab
           .to_hash
 
         if project.present?
-          processor = ::Gitlab::Ci::ExternalFiles::Processor.new(initial_config)
+          processor = ::Gitlab::Ci::ExternalFiles::Processor.new(initial_config, project)
           @config = processor.perform
         else
           @config = initial_config
diff --git a/lib/gitlab/ci/external_files/external_file.rb b/lib/gitlab/ci/external_files/external_file.rb
index 47d6d1c0fdc..45c6ca3fbfe 100644
--- a/lib/gitlab/ci/external_files/external_file.rb
+++ b/lib/gitlab/ci/external_files/external_file.rb
@@ -4,33 +4,38 @@ module Gitlab
   module Ci
     module ExternalFiles
       class ExternalFile
-        def initialize(value)
+        def initialize(value, project)
           @value = value
+          @project = project
         end
 
         def content
           if remote_url?
             open(value).read
           else
-            File.read(base_path)
+            local_file_content
           end
         end
 
         def valid?
-          remote_url? || File.exist?(base_path)
+          remote_url? || local_file_content
         end
 
         private
 
-        attr_reader :value
-
-        def base_path
-          "#{Rails.root}/#{value}"
-        end
+        attr_reader :value, :project
 
         def remote_url?
           ::Gitlab::UrlSanitizer.valid?(value)
         end
+
+        def local_file_content
+          project.repository.fetch_file_for(sha, value)
+        end
+
+        def sha
+          @sha ||= project.repository.commit.sha
+        end
       end
     end
   end
diff --git a/lib/gitlab/ci/external_files/mapper.rb b/lib/gitlab/ci/external_files/mapper.rb
index 4b5d2ddd6a1..2d40e296546 100644
--- a/lib/gitlab/ci/external_files/mapper.rb
+++ b/lib/gitlab/ci/external_files/mapper.rb
@@ -2,12 +2,12 @@ module Gitlab
   module Ci
     module ExternalFiles
       class Mapper
-        def self.fetch_paths(values)
-          paths = values.fetch(:includes, [])
-          normalize_paths(paths)
+        def initialize(values, project)
+          @paths = values.fetch(:includes, [])
+          @project = project
         end
 
-        def self.normalize_paths(paths)
+        def process
           if paths.is_a?(String)
             [build_external_file(paths)]
           else
@@ -15,8 +15,12 @@ module Gitlab
           end
         end
 
-        def self.build_external_file(path)
-          ::Gitlab::Ci::ExternalFiles::ExternalFile.new(path)
+        private
+
+        attr_reaer :paths, :project
+
+        def build_external_file(path)
+          ::Gitlab::Ci::ExternalFiles::ExternalFile.new(path, project)
         end
       end
     end
diff --git a/lib/gitlab/ci/external_files/processor.rb b/lib/gitlab/ci/external_files/processor.rb
index 221b6f58b98..ac51db05d8e 100644
--- a/lib/gitlab/ci/external_files/processor.rb
+++ b/lib/gitlab/ci/external_files/processor.rb
@@ -4,9 +4,9 @@ module Gitlab
       class Processor
         ExternalFileError = Class.new(StandardError)
 
-        def initialize(values)
+        def initialize(values, project)
           @values = values
-          @external_files = ::Gitlab::Ci::ExternalFiles::Mapper.fetch_paths(values)
+          @external_files = ::Gitlab::Ci::ExternalFiles::Mapper.fetch_paths(values, project).process
         end
 
         def perform
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 702bcd3802d..0f79fbede9f 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -73,13 +73,13 @@ module Gitlab
         end
       end
 
-      def self.validation_message(content, opts = {})
+      def self.validation_message(content, project = nil, opts = {})
         return 'Please provide content of .gitlab-ci.yml' if content.blank?
 
         begin
-          Gitlab::Ci::YamlProcessor.new(content, opts)
+          Gitlab::Ci::YamlProcessor.new(content, project, opts)
           nil
-        rescue ValidationError => e
+        rescue ValidationError, ::Gitlab::Ci::ExternalFiles::Processor::ExternalFileError => e
           e.message
         end
       end
-- 
cgit v1.2.1


From eca73d2b30a62876a3148bd1a8b1dfd6d48977fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Fri, 7 Sep 2018 21:33:06 +0200
Subject: Address MR comments

CE mirror of 1269dc47b7f8d1a9913de326c9bd356d3e603663
---
 lib/gitlab/ci/config.rb                       | 20 ++++++++++----------
 lib/gitlab/ci/external_files/external_file.rb |  8 +++++---
 lib/gitlab/ci/external_files/mapper.rb        |  4 ++--
 lib/gitlab/ci/external_files/processor.rb     |  6 +++---
 4 files changed, 20 insertions(+), 18 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index caa3a7c3c86..4a0d67720a9 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -7,17 +7,10 @@ module Gitlab
       ConfigError = Class.new(StandardError)
 
       def initialize(config, project = nil, opts = {})
-        initial_config = Config::Extendable
+        @config = Config::Extendable
           .new(build_config(config, opts))
           .to_hash
 
-        if project.present?
-          processor = ::Gitlab::Ci::ExternalFiles::Processor.new(initial_config, project)
-          @config = processor.perform
-        else
-          @config = initial_config
-        end
-
         @global = Entry::Global.new(@config)
         @global.compose!
       rescue Loader::FormatError, Extendable::ExtensionError => e
@@ -72,8 +65,15 @@ module Gitlab
       end
 
       # 'opts' argument is used in EE see /ee/lib/ee/gitlab/ci/config.rb
-      def build_config(config, opts = {})
-        Loader.new(config).load!
+      def build_config(config, project,  opts = {})
+        initial_config = Loader.new(config).load!
+
+        if project.present?
+          processor = ::Gitlab::Ci::ExternalFiles::Processor.new(initial_config, project)
+          processor.perform
+        else
+          initial_config
+        end
       end
     end
   end
diff --git a/lib/gitlab/ci/external_files/external_file.rb b/lib/gitlab/ci/external_files/external_file.rb
index 45c6ca3fbfe..f481e9b0a39 100644
--- a/lib/gitlab/ci/external_files/external_file.rb
+++ b/lib/gitlab/ci/external_files/external_file.rb
@@ -4,6 +4,8 @@ module Gitlab
   module Ci
     module ExternalFiles
       class ExternalFile
+        attr_reader :value, :project
+
         def initialize(value, project)
           @value = value
           @project = project
@@ -11,10 +13,12 @@ module Gitlab
 
         def content
           if remote_url?
-            open(value).read
+            HTTParty.get(value)
           else
             local_file_content
           end
+        rescue HTTParty::Error, Timeout::Error
+          nil
         end
 
         def valid?
@@ -23,8 +27,6 @@ module Gitlab
 
         private
 
-        attr_reader :value, :project
-
         def remote_url?
           ::Gitlab::UrlSanitizer.valid?(value)
         end
diff --git a/lib/gitlab/ci/external_files/mapper.rb b/lib/gitlab/ci/external_files/mapper.rb
index 2d40e296546..fa252cc3b9e 100644
--- a/lib/gitlab/ci/external_files/mapper.rb
+++ b/lib/gitlab/ci/external_files/mapper.rb
@@ -3,7 +3,7 @@ module Gitlab
     module ExternalFiles
       class Mapper
         def initialize(values, project)
-          @paths = values.fetch(:includes, [])
+          @paths = values.fetch(:include, [])
           @project = project
         end
 
@@ -17,7 +17,7 @@ module Gitlab
 
         private
 
-        attr_reaer :paths, :project
+        attr_reader :paths, :project
 
         def build_external_file(path)
           ::Gitlab::Ci::ExternalFiles::ExternalFile.new(path, project)
diff --git a/lib/gitlab/ci/external_files/processor.rb b/lib/gitlab/ci/external_files/processor.rb
index ac51db05d8e..5657f025084 100644
--- a/lib/gitlab/ci/external_files/processor.rb
+++ b/lib/gitlab/ci/external_files/processor.rb
@@ -6,7 +6,7 @@ module Gitlab
 
         def initialize(values, project)
           @values = values
-          @external_files = ::Gitlab::Ci::ExternalFiles::Mapper.fetch_paths(values, project).process
+          @external_files = ::Gitlab::Ci::ExternalFiles::Mapper.new(values, project).process
         end
 
         def perform
@@ -26,7 +26,7 @@ module Gitlab
 
         def validate_external_file(external_file)
           unless external_file.valid?
-            raise ExternalFileError, 'External files should be a valid local or remote file'
+            raise ExternalFileError, "External file: '#{external_file.value}' should be a valid local or remote file"
           end
         end
 
@@ -36,7 +36,7 @@ module Gitlab
         end
 
         def remove_include_keyword
-          values.delete(:includes)
+          values.delete(:include)
           values
         end
       end
-- 
cgit v1.2.1


From 1a53f017b4c2106da3425f3dcbe40fb95fd44bbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Fri, 7 Sep 2018 21:37:44 +0200
Subject: Make Repository#blob_data_at public

CE mirror of 17de13ada1a98da060802c55889489a512183cd1
---
 lib/gitlab/ci/external_files/external_file.rb | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/ci/external_files/external_file.rb b/lib/gitlab/ci/external_files/external_file.rb
index f481e9b0a39..4fffdc5f716 100644
--- a/lib/gitlab/ci/external_files/external_file.rb
+++ b/lib/gitlab/ci/external_files/external_file.rb
@@ -1,5 +1,3 @@
-require 'open-uri'
-
 module Gitlab
   module Ci
     module ExternalFiles
@@ -32,7 +30,7 @@ module Gitlab
         end
 
         def local_file_content
-          project.repository.fetch_file_for(sha, value)
+          project.repository.blob_data_at(sha, value)
         end
 
         def sha
-- 
cgit v1.2.1


From 3149b5cfea4d8f14d69bb2520974f588454a0762 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Fri, 7 Sep 2018 22:03:05 +0200
Subject: Improve external architecture

CE mirror of 4f17c7b2c30188302e6a73421acbf5a09fb2c823
---
 lib/gitlab/ci/config.rb                       | 16 +++++----
 lib/gitlab/ci/external/file/local.rb          | 35 +++++++++++++++++++
 lib/gitlab/ci/external/file/remote.rb         | 25 ++++++++++++++
 lib/gitlab/ci/external/mapper.rb              | 31 +++++++++++++++++
 lib/gitlab/ci/external/processor.rb           | 50 +++++++++++++++++++++++++++
 lib/gitlab/ci/external_files/external_file.rb | 42 ----------------------
 lib/gitlab/ci/external_files/mapper.rb        | 28 ---------------
 lib/gitlab/ci/external_files/processor.rb     | 45 ------------------------
 lib/gitlab/ci/yaml_processor.rb               |  8 ++---
 9 files changed, 155 insertions(+), 125 deletions(-)
 create mode 100644 lib/gitlab/ci/external/file/local.rb
 create mode 100644 lib/gitlab/ci/external/file/remote.rb
 create mode 100644 lib/gitlab/ci/external/mapper.rb
 create mode 100644 lib/gitlab/ci/external/processor.rb
 delete mode 100644 lib/gitlab/ci/external_files/external_file.rb
 delete mode 100644 lib/gitlab/ci/external_files/mapper.rb
 delete mode 100644 lib/gitlab/ci/external_files/processor.rb

(limited to 'lib')

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 4a0d67720a9..1779fbb2a39 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -6,7 +6,7 @@ module Gitlab
     class Config
       ConfigError = Class.new(StandardError)
 
-      def initialize(config, project = nil, opts = {})
+      def initialize(config, opts = {})
         @config = Config::Extendable
           .new(build_config(config, opts))
           .to_hash
@@ -64,17 +64,21 @@ module Gitlab
         @global.jobs_value
       end
 
-      # 'opts' argument is used in EE see /ee/lib/ee/gitlab/ci/config.rb
-      def build_config(config, project,  opts = {})
+      def build_config(config, opts = {})
         initial_config = Loader.new(config).load!
+        project = opts.fetch(:project, nil)
 
-        if project.present?
-          processor = ::Gitlab::Ci::ExternalFiles::Processor.new(initial_config, project)
-          processor.perform
+        if project
+          process_external_files(initial_config, project, opts)
         else
           initial_config
         end
       end
+
+      def process_external_files(config, project, opts)
+        branch_name = opts.fetch(:branch_name, project.default_branch)
+        ::Gitlab::Ci::External::Processor.new(config, project, branch_name).perform
+      end
     end
   end
 end
diff --git a/lib/gitlab/ci/external/file/local.rb b/lib/gitlab/ci/external/file/local.rb
new file mode 100644
index 00000000000..dc1ba923411
--- /dev/null
+++ b/lib/gitlab/ci/external/file/local.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module Ci
+    module External
+      module File
+        class Local
+          attr_reader :value, :project, :branch_name
+
+          def initialize(value, project, branch_name)
+            @value = value
+            @project = project
+            @branch_name = branch_name
+          end
+
+          def valid?
+            commit && local_file_content
+          end
+
+          def content
+            local_file_content
+          end
+
+          private
+
+          def commit
+            @commit ||= project.repository.commit(branch_name)
+          end
+
+          def local_file_content
+            @local_file_content ||= project.repository.blob_data_at(commit.sha, value)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external/file/remote.rb b/lib/gitlab/ci/external/file/remote.rb
new file mode 100644
index 00000000000..7d8cd1957ad
--- /dev/null
+++ b/lib/gitlab/ci/external/file/remote.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module Ci
+    module External
+      module File
+        class Remote
+          attr_reader :value
+
+          def initialize(value)
+            @value = value
+          end
+
+          def valid?
+            ::Gitlab::UrlSanitizer.valid?(value) && content
+          end
+
+          def content
+            HTTParty.get(value)
+          rescue HTTParty::Error, Timeout::Error
+            false
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external/mapper.rb b/lib/gitlab/ci/external/mapper.rb
new file mode 100644
index 00000000000..f2e5ec972df
--- /dev/null
+++ b/lib/gitlab/ci/external/mapper.rb
@@ -0,0 +1,31 @@
+module Gitlab
+  module Ci
+    module External
+      class Mapper
+        def initialize(values, project, branch_name)
+          @paths = Array(values.fetch(:include, []))
+          @project = project
+          @branch_name = branch_name
+        end
+
+        def process
+          paths.map { |path| build_external_file(path) }
+        end
+
+        private
+
+        attr_reader :paths, :project, :branch_name
+
+        def build_external_file(path)
+          remote_file = Gitlab::Ci::External::File::Remote.new(path)
+
+          if remote_file.valid?
+            remote_file
+          else
+            ::Gitlab::Ci::External::File::Local.new(path, project, branch_name)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external/processor.rb b/lib/gitlab/ci/external/processor.rb
new file mode 100644
index 00000000000..5cae0166346
--- /dev/null
+++ b/lib/gitlab/ci/external/processor.rb
@@ -0,0 +1,50 @@
+module Gitlab
+  module Ci
+    module External
+      class Processor
+        FileError = Class.new(StandardError)
+
+        def initialize(values, project, branch_name)
+          @values = values
+          @external_files = ::Gitlab::Ci::External::Mapper.new(values, project, branch_name).process
+          @content = {}
+        end
+
+        def perform
+          return values if external_files.empty?
+
+          external_files.each do |external_file|
+            validate_external_file(external_file)
+            @content.merge!(content_of(external_file))
+          end
+
+          append_external_content
+          remove_include_keyword
+        end
+
+        private
+
+        attr_reader :values, :external_files, :content
+
+        def validate_external_file(external_file)
+          unless external_file.valid?
+            raise FileError, "External file: '#{external_file.value}' should be a valid local or remote file"
+          end
+        end
+
+        def content_of(external_file)
+          ::Gitlab::Ci::Config::Loader.new(external_file.content).load!
+        end
+
+        def append_external_content
+          @content.merge!(@values)
+        end
+
+        def remove_include_keyword
+          content.delete(:include)
+          content
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external_files/external_file.rb b/lib/gitlab/ci/external_files/external_file.rb
deleted file mode 100644
index 4fffdc5f716..00000000000
--- a/lib/gitlab/ci/external_files/external_file.rb
+++ /dev/null
@@ -1,42 +0,0 @@
-module Gitlab
-  module Ci
-    module ExternalFiles
-      class ExternalFile
-        attr_reader :value, :project
-
-        def initialize(value, project)
-          @value = value
-          @project = project
-        end
-
-        def content
-          if remote_url?
-            HTTParty.get(value)
-          else
-            local_file_content
-          end
-        rescue HTTParty::Error, Timeout::Error
-          nil
-        end
-
-        def valid?
-          remote_url? || local_file_content
-        end
-
-        private
-
-        def remote_url?
-          ::Gitlab::UrlSanitizer.valid?(value)
-        end
-
-        def local_file_content
-          project.repository.blob_data_at(sha, value)
-        end
-
-        def sha
-          @sha ||= project.repository.commit.sha
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/external_files/mapper.rb b/lib/gitlab/ci/external_files/mapper.rb
deleted file mode 100644
index fa252cc3b9e..00000000000
--- a/lib/gitlab/ci/external_files/mapper.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-module Gitlab
-  module Ci
-    module ExternalFiles
-      class Mapper
-        def initialize(values, project)
-          @paths = values.fetch(:include, [])
-          @project = project
-        end
-
-        def process
-          if paths.is_a?(String)
-            [build_external_file(paths)]
-          else
-            paths.map { |path| build_external_file(path) }
-          end
-        end
-
-        private
-
-        attr_reader :paths, :project
-
-        def build_external_file(path)
-          ::Gitlab::Ci::ExternalFiles::ExternalFile.new(path, project)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/external_files/processor.rb b/lib/gitlab/ci/external_files/processor.rb
deleted file mode 100644
index 5657f025084..00000000000
--- a/lib/gitlab/ci/external_files/processor.rb
+++ /dev/null
@@ -1,45 +0,0 @@
-module Gitlab
-  module Ci
-    module ExternalFiles
-      class Processor
-        ExternalFileError = Class.new(StandardError)
-
-        def initialize(values, project)
-          @values = values
-          @external_files = ::Gitlab::Ci::ExternalFiles::Mapper.new(values, project).process
-        end
-
-        def perform
-          return values if external_files.empty?
-
-          external_files.each do |external_file|
-            validate_external_file(external_file)
-            append_external_content(external_file)
-          end
-
-          remove_include_keyword
-        end
-
-        private
-
-        attr_reader :values, :external_files
-
-        def validate_external_file(external_file)
-          unless external_file.valid?
-            raise ExternalFileError, "External file: '#{external_file.value}' should be a valid local or remote file"
-          end
-        end
-
-        def append_external_content(external_file)
-          external_values = ::Gitlab::Ci::Config::Loader.new(external_file.content).load!
-          @values.merge!(external_values)
-        end
-
-        def remove_include_keyword
-          values.delete(:include)
-          values
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 0f79fbede9f..67f5d2d6ae4 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -7,8 +7,8 @@ module Gitlab
 
       attr_reader :cache, :stages, :jobs
 
-      def initialize(config, project = nil, opts = {})
-        @ci_config = Gitlab::Ci::Config.new(config, project, opts)
+      def initialize(config, opts = {})
+        @ci_config = Gitlab::Ci::Config.new(config, opts)
         @config = @ci_config.to_hash
 
         unless @ci_config.valid?
@@ -73,11 +73,11 @@ module Gitlab
         end
       end
 
-      def self.validation_message(content, project = nil, opts = {})
+      def self.validation_message(content, opts = {})
         return 'Please provide content of .gitlab-ci.yml' if content.blank?
 
         begin
-          Gitlab::Ci::YamlProcessor.new(content, project, opts)
+          Gitlab::Ci::YamlProcessor.new(content, opts)
           nil
         rescue ValidationError, ::Gitlab::Ci::ExternalFiles::Processor::ExternalFileError => e
           e.message
-- 
cgit v1.2.1


From 1e816a972164c515bf99c39bd8880dd23e534c9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Fri, 7 Sep 2018 22:22:52 +0200
Subject: Address MR suggestions

CE mirror of c4578b951e331fe8c75cd4f948ce74cec6587bad
---
 lib/gitlab/ci/config.rb               |  4 ++--
 lib/gitlab/ci/external/file/local.rb  | 18 +++++++-----------
 lib/gitlab/ci/external/file/remote.rb | 18 +++++++++++-------
 lib/gitlab/ci/external/mapper.rb      | 17 +++++++++--------
 lib/gitlab/ci/external/processor.rb   | 10 +++++-----
 5 files changed, 34 insertions(+), 33 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 1779fbb2a39..19d93c6e785 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -76,8 +76,8 @@ module Gitlab
       end
 
       def process_external_files(config, project, opts)
-        branch_name = opts.fetch(:branch_name, project.default_branch)
-        ::Gitlab::Ci::External::Processor.new(config, project, branch_name).perform
+        sha = opts.fetch(:sha, project.repository.commit.sha)
+          ::Gitlab::Ci::External::Processor.new(config, project, sha).perform
       end
     end
   end
diff --git a/lib/gitlab/ci/external/file/local.rb b/lib/gitlab/ci/external/file/local.rb
index dc1ba923411..27e827222e5 100644
--- a/lib/gitlab/ci/external/file/local.rb
+++ b/lib/gitlab/ci/external/file/local.rb
@@ -3,16 +3,16 @@ module Gitlab
     module External
       module File
         class Local
-          attr_reader :value, :project, :branch_name
+          attr_reader :location, :project, :branch_name
 
-          def initialize(value, project, branch_name)
-            @value = value
-            @project = project
-            @branch_name = branch_name
+          def initialize(location, opts = {})
+            @location = location
+            @project = opts.fetch(:project)
+            @sha = opts.fetch(:sha)
           end
 
           def valid?
-            commit && local_file_content
+            local_file_content
           end
 
           def content
@@ -21,12 +21,8 @@ module Gitlab
 
           private
 
-          def commit
-            @commit ||= project.repository.commit(branch_name)
-          end
-
           def local_file_content
-            @local_file_content ||= project.repository.blob_data_at(commit.sha, value)
+            @local_file_content ||= project.repository.blob_data_at(sha, location)
           end
         end
       end
diff --git a/lib/gitlab/ci/external/file/remote.rb b/lib/gitlab/ci/external/file/remote.rb
index 7d8cd1957ad..aa089860525 100644
--- a/lib/gitlab/ci/external/file/remote.rb
+++ b/lib/gitlab/ci/external/file/remote.rb
@@ -3,20 +3,24 @@ module Gitlab
     module External
       module File
         class Remote
-          attr_reader :value
+          attr_reader :location
 
-          def initialize(value)
-            @value = value
+          def initialize(location, opts = {})
+            @location = location
           end
 
           def valid?
-            ::Gitlab::UrlSanitizer.valid?(value) && content
+            ::Gitlab::UrlSanitizer.valid?(location) && content
           end
 
           def content
-            HTTParty.get(value)
-          rescue HTTParty::Error, Timeout::Error
-            false
+            return @content if defined?(@content)
+
+            @content ||= begin
+                           HTTParty.get(location)
+                         rescue HTTParty::Error, Timeout::Error
+                           false
+                         end
           end
         end
       end
diff --git a/lib/gitlab/ci/external/mapper.rb b/lib/gitlab/ci/external/mapper.rb
index f2e5ec972df..cc9be317ebe 100644
--- a/lib/gitlab/ci/external/mapper.rb
+++ b/lib/gitlab/ci/external/mapper.rb
@@ -2,27 +2,28 @@ module Gitlab
   module Ci
     module External
       class Mapper
-        def initialize(values, project, branch_name)
-          @paths = Array(values.fetch(:include, []))
+        def initialize(values, project, sha)
+          @locations = Array(values.fetch(:include, []))
           @project = project
-          @branch_name = branch_name
+          @sha = sha
         end
 
         def process
-          paths.map { |path| build_external_file(path) }
+          locations.map { |location| build_external_file(location) }
         end
 
         private
 
-        attr_reader :paths, :project, :branch_name
+        attr_reader :locations, :project, :sha
 
-        def build_external_file(path)
-          remote_file = Gitlab::Ci::External::File::Remote.new(path)
+        def build_external_file(location)
+          remote_file = Gitlab::Ci::External::File::Remote.new(location)
 
           if remote_file.valid?
             remote_file
           else
-            ::Gitlab::Ci::External::File::Local.new(path, project, branch_name)
+            options = { project: project, sha: sha }
+            Gitlab::Ci::External::File::Local.new(location, options)
           end
         end
       end
diff --git a/lib/gitlab/ci/external/processor.rb b/lib/gitlab/ci/external/processor.rb
index 5cae0166346..44dc3183367 100644
--- a/lib/gitlab/ci/external/processor.rb
+++ b/lib/gitlab/ci/external/processor.rb
@@ -4,9 +4,9 @@ module Gitlab
       class Processor
         FileError = Class.new(StandardError)
 
-        def initialize(values, project, branch_name)
+        def initialize(values, project, sha)
           @values = values
-          @external_files = ::Gitlab::Ci::External::Mapper.new(values, project, branch_name).process
+          @external_files = Gitlab::Ci::External::Mapper.new(values, project, sha).process
           @content = {}
         end
 
@@ -18,7 +18,7 @@ module Gitlab
             @content.merge!(content_of(external_file))
           end
 
-          append_external_content
+          append_inline_content
           remove_include_keyword
         end
 
@@ -28,7 +28,7 @@ module Gitlab
 
         def validate_external_file(external_file)
           unless external_file.valid?
-            raise FileError, "External file: '#{external_file.value}' should be a valid local or remote file"
+            raise FileError, "External file: '#{external_file.location}' should be a valid local or remote file"
           end
         end
 
@@ -36,7 +36,7 @@ module Gitlab
           ::Gitlab::Ci::Config::Loader.new(external_file.content).load!
         end
 
-        def append_external_content
+        def append_inline_content
           @content.merge!(@values)
         end
 
-- 
cgit v1.2.1


From 6a8133b943a8e06571f5497bea0f36c236b2bf90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Fri, 7 Sep 2018 22:37:37 +0200
Subject: Stub http request on specs intead of mocking HTTParty

CE mirror of bb2a9fde8e6a4d1df13638fe336f641b9c72ef59
---
 lib/gitlab/ci/external/file/remote.rb | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/ci/external/file/remote.rb b/lib/gitlab/ci/external/file/remote.rb
index aa089860525..e728e3de77d 100644
--- a/lib/gitlab/ci/external/file/remote.rb
+++ b/lib/gitlab/ci/external/file/remote.rb
@@ -3,6 +3,7 @@ module Gitlab
     module External
       module File
         class Remote
+          include Gitlab::Utils::StrongMemoize
           attr_reader :location
 
           def initialize(location, opts = {})
@@ -16,11 +17,13 @@ module Gitlab
           def content
             return @content if defined?(@content)
 
-            @content ||= begin
-                           HTTParty.get(location)
-                         rescue HTTParty::Error, Timeout::Error
-                           false
-                         end
+            @content = strong_memoize(:content) do
+              begin
+                HTTParty.get(location)
+              rescue HTTParty::Error, Timeout::Error
+                false
+              end
+            end
           end
         end
       end
-- 
cgit v1.2.1


From 797046e358bcf0fcd8cab413fcccad1614180aa9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Sat, 8 Sep 2018 12:09:14 +0200
Subject: Reconcile differences in lib/gitlab/ci/external/file

---
 lib/gitlab/ci/external/file/base.rb   | 27 +++++++++++++++++++++++++++
 lib/gitlab/ci/external/file/local.rb  | 19 ++++++++++---------
 lib/gitlab/ci/external/file/remote.rb | 20 ++++++++------------
 3 files changed, 45 insertions(+), 21 deletions(-)
 create mode 100644 lib/gitlab/ci/external/file/base.rb

(limited to 'lib')

diff --git a/lib/gitlab/ci/external/file/base.rb b/lib/gitlab/ci/external/file/base.rb
new file mode 100644
index 00000000000..4591b3ec82e
--- /dev/null
+++ b/lib/gitlab/ci/external/file/base.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Ci
+    module External
+      module File
+        class Base
+          YAML_WHITELIST_EXTENSION = /(yml|yaml)$/i.freeze
+
+          def initialize(location, opts = {})
+            @location = location
+          end
+
+          def valid?
+            location.match(YAML_WHITELIST_EXTENSION) && content
+          end
+
+          def content
+            raise NotImplementedError, 'content must be implemented and return a string or nil'
+          end
+
+          def error_message
+            raise NotImplementedError, 'error_message must be implemented and return a string'
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/external/file/local.rb b/lib/gitlab/ci/external/file/local.rb
index 27e827222e5..829cc9b3d19 100644
--- a/lib/gitlab/ci/external/file/local.rb
+++ b/lib/gitlab/ci/external/file/local.rb
@@ -2,27 +2,28 @@ module Gitlab
   module Ci
     module External
       module File
-        class Local
-          attr_reader :location, :project, :branch_name
+        class Local < Base
+          attr_reader :location, :project, :sha
 
           def initialize(location, opts = {})
-            @location = location
+            super
+
             @project = opts.fetch(:project)
             @sha = opts.fetch(:sha)
           end
 
-          def valid?
-            local_file_content
+          def content
+            @content ||= fetch_local_content
           end
 
-          def content
-            local_file_content
+          def error_message
+            "Local file '#{location}' is not valid."
           end
 
           private
 
-          def local_file_content
-            @local_file_content ||= project.repository.blob_data_at(sha, location)
+          def fetch_local_content
+            project.repository.blob_data_at(sha, location)
           end
         end
       end
diff --git a/lib/gitlab/ci/external/file/remote.rb b/lib/gitlab/ci/external/file/remote.rb
index e728e3de77d..b4294231265 100644
--- a/lib/gitlab/ci/external/file/remote.rb
+++ b/lib/gitlab/ci/external/file/remote.rb
@@ -2,29 +2,25 @@ module Gitlab
   module Ci
     module External
       module File
-        class Remote
+        class Remote < Base
           include Gitlab::Utils::StrongMemoize
           attr_reader :location
 
-          def initialize(location, opts = {})
-            @location = location
-          end
-
-          def valid?
-            ::Gitlab::UrlSanitizer.valid?(location) && content
-          end
-
           def content
             return @content if defined?(@content)
 
             @content = strong_memoize(:content) do
               begin
-                HTTParty.get(location)
-              rescue HTTParty::Error, Timeout::Error
-                false
+                Gitlab::HTTP.get(location)
+              rescue Gitlab::HTTP::Error, Timeout::Error, SocketError, Gitlab::HTTP::BlockedUrlError
+                nil
               end
             end
           end
+
+          def error_message
+            "Remote file '#{location}' is not valid."
+          end
         end
       end
     end
-- 
cgit v1.2.1


From 5e0ce238d2c19d11fac5dd40b922b01f134ffcb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Sat, 8 Sep 2018 14:09:26 +0200
Subject: Reconcile differences in lib/gitlab/ci/external

---
 lib/gitlab/ci/external/mapper.rb    | 6 ++----
 lib/gitlab/ci/external/processor.rb | 8 ++++----
 2 files changed, 6 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/ci/external/mapper.rb b/lib/gitlab/ci/external/mapper.rb
index cc9be317ebe..3c359efa803 100644
--- a/lib/gitlab/ci/external/mapper.rb
+++ b/lib/gitlab/ci/external/mapper.rb
@@ -17,10 +17,8 @@ module Gitlab
         attr_reader :locations, :project, :sha
 
         def build_external_file(location)
-          remote_file = Gitlab::Ci::External::File::Remote.new(location)
-
-          if remote_file.valid?
-            remote_file
+          if ::Gitlab::UrlSanitizer.valid?(location)
+            Gitlab::Ci::External::File::Remote.new(location)
           else
             options = { project: project, sha: sha }
             Gitlab::Ci::External::File::Local.new(location, options)
diff --git a/lib/gitlab/ci/external/processor.rb b/lib/gitlab/ci/external/processor.rb
index 44dc3183367..22588867a08 100644
--- a/lib/gitlab/ci/external/processor.rb
+++ b/lib/gitlab/ci/external/processor.rb
@@ -15,7 +15,7 @@ module Gitlab
 
           external_files.each do |external_file|
             validate_external_file(external_file)
-            @content.merge!(content_of(external_file))
+            @content.deep_merge!(content_of(external_file))
           end
 
           append_inline_content
@@ -28,16 +28,16 @@ module Gitlab
 
         def validate_external_file(external_file)
           unless external_file.valid?
-            raise FileError, "External file: '#{external_file.location}' should be a valid local or remote file"
+            raise FileError, external_file.error_message
           end
         end
 
         def content_of(external_file)
-          ::Gitlab::Ci::Config::Loader.new(external_file.content).load!
+          Gitlab::Ci::Config::Loader.new(external_file.content).load!
         end
 
         def append_inline_content
-          @content.merge!(@values)
+          @content.deep_merge!(@values)
         end
 
         def remove_include_keyword
-- 
cgit v1.2.1


From df16969c24c608682b8b46883a8e6e20147a3791 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Sat, 8 Sep 2018 14:55:36 +0200
Subject: Reconcile differences in lib/gitlab/ci

---
 lib/gitlab/ci/config.rb         | 26 ++++++++++++++++----------
 lib/gitlab/ci/yaml_processor.rb |  2 +-
 2 files changed, 17 insertions(+), 11 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 19d93c6e785..0ffd1791791 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -7,14 +7,18 @@ module Gitlab
       ConfigError = Class.new(StandardError)
 
       def initialize(config, opts = {})
-        @config = Config::Extendable
-          .new(build_config(config, opts))
-          .to_hash
-
-        @global = Entry::Global.new(@config)
-        @global.compose!
-      rescue Loader::FormatError, Extendable::ExtensionError => e
-        raise Config::ConfigError, e.message
+        begin
+          @config = Config::Extendable
+            .new(build_config(config, opts))
+            .to_hash
+
+          @global = Entry::Global.new(@config)
+          @global.compose!
+        rescue Loader::FormatError, Extendable::ExtensionError => e
+          raise Config::ConfigError, e.message
+        end
+      rescue ::Gitlab::Ci::External::Processor::FileError => e
+        raise ::Gitlab::Ci::YamlProcessor::ValidationError, e.message
       end
 
       def valid?
@@ -64,6 +68,8 @@ module Gitlab
         @global.jobs_value
       end
 
+      private
+
       def build_config(config, opts = {})
         initial_config = Loader.new(config).load!
         project = opts.fetch(:project, nil)
@@ -76,8 +82,8 @@ module Gitlab
       end
 
       def process_external_files(config, project, opts)
-        sha = opts.fetch(:sha, project.repository.commit.sha)
-          ::Gitlab::Ci::External::Processor.new(config, project, sha).perform
+        sha = opts.fetch(:sha) { project.repository.root_ref_sha }
+        ::Gitlab::Ci::External::Processor.new(config, project, sha).perform
       end
     end
   end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 67f5d2d6ae4..5d1864ae9e2 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -79,7 +79,7 @@ module Gitlab
         begin
           Gitlab::Ci::YamlProcessor.new(content, opts)
           nil
-        rescue ValidationError, ::Gitlab::Ci::ExternalFiles::Processor::ExternalFileError => e
+        rescue ValidationError => e
           e.message
         end
       end
-- 
cgit v1.2.1


From b470f975e18ce792e78026b7aaa2c4a4fa45e237 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Tue, 11 Sep 2018 16:26:57 +0200
Subject: Add missing frozen string literal comments

---
 lib/gitlab/ci/external/file/base.rb   | 2 ++
 lib/gitlab/ci/external/file/local.rb  | 2 ++
 lib/gitlab/ci/external/file/remote.rb | 2 ++
 lib/gitlab/ci/external/mapper.rb      | 2 ++
 lib/gitlab/ci/external/processor.rb   | 2 ++
 5 files changed, 10 insertions(+)

(limited to 'lib')

diff --git a/lib/gitlab/ci/external/file/base.rb b/lib/gitlab/ci/external/file/base.rb
index 4591b3ec82e..f4da07b0b02 100644
--- a/lib/gitlab/ci/external/file/base.rb
+++ b/lib/gitlab/ci/external/file/base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module External
diff --git a/lib/gitlab/ci/external/file/local.rb b/lib/gitlab/ci/external/file/local.rb
index 829cc9b3d19..1aa7f687507 100644
--- a/lib/gitlab/ci/external/file/local.rb
+++ b/lib/gitlab/ci/external/file/local.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module External
diff --git a/lib/gitlab/ci/external/file/remote.rb b/lib/gitlab/ci/external/file/remote.rb
index b4294231265..59bb3e8999e 100644
--- a/lib/gitlab/ci/external/file/remote.rb
+++ b/lib/gitlab/ci/external/file/remote.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module External
diff --git a/lib/gitlab/ci/external/mapper.rb b/lib/gitlab/ci/external/mapper.rb
index 3c359efa803..58bd6a19acf 100644
--- a/lib/gitlab/ci/external/mapper.rb
+++ b/lib/gitlab/ci/external/mapper.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module External
diff --git a/lib/gitlab/ci/external/processor.rb b/lib/gitlab/ci/external/processor.rb
index 22588867a08..76cf3ce89f9 100644
--- a/lib/gitlab/ci/external/processor.rb
+++ b/lib/gitlab/ci/external/processor.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module External
-- 
cgit v1.2.1


From 7600a6469bbd2975c80b48e6bb9b95af1e40795f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Tue, 11 Sep 2018 16:29:48 +0200
Subject: Stack rescue blocks for Gitlab::Ci::Config#initialize

---
 lib/gitlab/ci/config.rb | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 0ffd1791791..fe98d25af29 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -7,16 +7,14 @@ module Gitlab
       ConfigError = Class.new(StandardError)
 
       def initialize(config, opts = {})
-        begin
-          @config = Config::Extendable
-            .new(build_config(config, opts))
-            .to_hash
-
-          @global = Entry::Global.new(@config)
-          @global.compose!
-        rescue Loader::FormatError, Extendable::ExtensionError => e
-          raise Config::ConfigError, e.message
-        end
+        @config = Config::Extendable
+          .new(build_config(config, opts))
+          .to_hash
+
+        @global = Entry::Global.new(@config)
+        @global.compose!
+      rescue Loader::FormatError, Extendable::ExtensionError => e
+        raise Config::ConfigError, e.message
       rescue ::Gitlab::Ci::External::Processor::FileError => e
         raise ::Gitlab::Ci::YamlProcessor::ValidationError, e.message
       end
-- 
cgit v1.2.1