From 3149b5cfea4d8f14d69bb2520974f588454a0762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Fri, 7 Sep 2018 22:03:05 +0200 Subject: Improve external architecture CE mirror of 4f17c7b2c30188302e6a73421acbf5a09fb2c823 --- lib/gitlab/ci/config.rb | 16 +++++---- lib/gitlab/ci/external/file/local.rb | 35 +++++++++++++++++++ lib/gitlab/ci/external/file/remote.rb | 25 ++++++++++++++ lib/gitlab/ci/external/mapper.rb | 31 +++++++++++++++++ lib/gitlab/ci/external/processor.rb | 50 +++++++++++++++++++++++++++ lib/gitlab/ci/external_files/external_file.rb | 42 ---------------------- lib/gitlab/ci/external_files/mapper.rb | 28 --------------- lib/gitlab/ci/external_files/processor.rb | 45 ------------------------ lib/gitlab/ci/yaml_processor.rb | 8 ++--- 9 files changed, 155 insertions(+), 125 deletions(-) create mode 100644 lib/gitlab/ci/external/file/local.rb create mode 100644 lib/gitlab/ci/external/file/remote.rb create mode 100644 lib/gitlab/ci/external/mapper.rb create mode 100644 lib/gitlab/ci/external/processor.rb delete mode 100644 lib/gitlab/ci/external_files/external_file.rb delete mode 100644 lib/gitlab/ci/external_files/mapper.rb delete mode 100644 lib/gitlab/ci/external_files/processor.rb (limited to 'lib') diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 4a0d67720a9..1779fbb2a39 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -6,7 +6,7 @@ module Gitlab class Config ConfigError = Class.new(StandardError) - def initialize(config, project = nil, opts = {}) + def initialize(config, opts = {}) @config = Config::Extendable .new(build_config(config, opts)) .to_hash @@ -64,17 +64,21 @@ module Gitlab @global.jobs_value end - # 'opts' argument is used in EE see /ee/lib/ee/gitlab/ci/config.rb - def build_config(config, project, opts = {}) + def build_config(config, opts = {}) initial_config = Loader.new(config).load! + project = opts.fetch(:project, nil) - if project.present? - processor = ::Gitlab::Ci::ExternalFiles::Processor.new(initial_config, project) - processor.perform + if project + process_external_files(initial_config, project, opts) else initial_config end end + + def process_external_files(config, project, opts) + branch_name = opts.fetch(:branch_name, project.default_branch) + ::Gitlab::Ci::External::Processor.new(config, project, branch_name).perform + end end end end diff --git a/lib/gitlab/ci/external/file/local.rb b/lib/gitlab/ci/external/file/local.rb new file mode 100644 index 00000000000..dc1ba923411 --- /dev/null +++ b/lib/gitlab/ci/external/file/local.rb @@ -0,0 +1,35 @@ +module Gitlab + module Ci + module External + module File + class Local + attr_reader :value, :project, :branch_name + + def initialize(value, project, branch_name) + @value = value + @project = project + @branch_name = branch_name + end + + def valid? + commit && local_file_content + end + + def content + local_file_content + end + + private + + def commit + @commit ||= project.repository.commit(branch_name) + end + + def local_file_content + @local_file_content ||= project.repository.blob_data_at(commit.sha, value) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/file/remote.rb b/lib/gitlab/ci/external/file/remote.rb new file mode 100644 index 00000000000..7d8cd1957ad --- /dev/null +++ b/lib/gitlab/ci/external/file/remote.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module External + module File + class Remote + attr_reader :value + + def initialize(value) + @value = value + end + + def valid? + ::Gitlab::UrlSanitizer.valid?(value) && content + end + + def content + HTTParty.get(value) + rescue HTTParty::Error, Timeout::Error + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/mapper.rb b/lib/gitlab/ci/external/mapper.rb new file mode 100644 index 00000000000..f2e5ec972df --- /dev/null +++ b/lib/gitlab/ci/external/mapper.rb @@ -0,0 +1,31 @@ +module Gitlab + module Ci + module External + class Mapper + def initialize(values, project, branch_name) + @paths = Array(values.fetch(:include, [])) + @project = project + @branch_name = branch_name + end + + def process + paths.map { |path| build_external_file(path) } + end + + private + + attr_reader :paths, :project, :branch_name + + def build_external_file(path) + remote_file = Gitlab::Ci::External::File::Remote.new(path) + + if remote_file.valid? + remote_file + else + ::Gitlab::Ci::External::File::Local.new(path, project, branch_name) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/processor.rb b/lib/gitlab/ci/external/processor.rb new file mode 100644 index 00000000000..5cae0166346 --- /dev/null +++ b/lib/gitlab/ci/external/processor.rb @@ -0,0 +1,50 @@ +module Gitlab + module Ci + module External + class Processor + FileError = Class.new(StandardError) + + def initialize(values, project, branch_name) + @values = values + @external_files = ::Gitlab::Ci::External::Mapper.new(values, project, branch_name).process + @content = {} + end + + def perform + return values if external_files.empty? + + external_files.each do |external_file| + validate_external_file(external_file) + @content.merge!(content_of(external_file)) + end + + append_external_content + remove_include_keyword + end + + private + + attr_reader :values, :external_files, :content + + def validate_external_file(external_file) + unless external_file.valid? + raise FileError, "External file: '#{external_file.value}' should be a valid local or remote file" + end + end + + def content_of(external_file) + ::Gitlab::Ci::Config::Loader.new(external_file.content).load! + end + + def append_external_content + @content.merge!(@values) + end + + def remove_include_keyword + content.delete(:include) + content + end + end + end + end +end diff --git a/lib/gitlab/ci/external_files/external_file.rb b/lib/gitlab/ci/external_files/external_file.rb deleted file mode 100644 index 4fffdc5f716..00000000000 --- a/lib/gitlab/ci/external_files/external_file.rb +++ /dev/null @@ -1,42 +0,0 @@ -module Gitlab - module Ci - module ExternalFiles - class ExternalFile - attr_reader :value, :project - - def initialize(value, project) - @value = value - @project = project - end - - def content - if remote_url? - HTTParty.get(value) - else - local_file_content - end - rescue HTTParty::Error, Timeout::Error - nil - end - - def valid? - remote_url? || local_file_content - end - - private - - def remote_url? - ::Gitlab::UrlSanitizer.valid?(value) - end - - def local_file_content - project.repository.blob_data_at(sha, value) - end - - def sha - @sha ||= project.repository.commit.sha - end - end - end - end -end diff --git a/lib/gitlab/ci/external_files/mapper.rb b/lib/gitlab/ci/external_files/mapper.rb deleted file mode 100644 index fa252cc3b9e..00000000000 --- a/lib/gitlab/ci/external_files/mapper.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Gitlab - module Ci - module ExternalFiles - class Mapper - def initialize(values, project) - @paths = values.fetch(:include, []) - @project = project - end - - def process - if paths.is_a?(String) - [build_external_file(paths)] - else - paths.map { |path| build_external_file(path) } - end - end - - private - - attr_reader :paths, :project - - def build_external_file(path) - ::Gitlab::Ci::ExternalFiles::ExternalFile.new(path, project) - end - end - end - end -end diff --git a/lib/gitlab/ci/external_files/processor.rb b/lib/gitlab/ci/external_files/processor.rb deleted file mode 100644 index 5657f025084..00000000000 --- a/lib/gitlab/ci/external_files/processor.rb +++ /dev/null @@ -1,45 +0,0 @@ -module Gitlab - module Ci - module ExternalFiles - class Processor - ExternalFileError = Class.new(StandardError) - - def initialize(values, project) - @values = values - @external_files = ::Gitlab::Ci::ExternalFiles::Mapper.new(values, project).process - end - - def perform - return values if external_files.empty? - - external_files.each do |external_file| - validate_external_file(external_file) - append_external_content(external_file) - end - - remove_include_keyword - end - - private - - attr_reader :values, :external_files - - def validate_external_file(external_file) - unless external_file.valid? - raise ExternalFileError, "External file: '#{external_file.value}' should be a valid local or remote file" - end - end - - def append_external_content(external_file) - external_values = ::Gitlab::Ci::Config::Loader.new(external_file.content).load! - @values.merge!(external_values) - end - - def remove_include_keyword - values.delete(:include) - values - end - end - end - end -end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 0f79fbede9f..67f5d2d6ae4 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -7,8 +7,8 @@ module Gitlab attr_reader :cache, :stages, :jobs - def initialize(config, project = nil, opts = {}) - @ci_config = Gitlab::Ci::Config.new(config, project, opts) + def initialize(config, opts = {}) + @ci_config = Gitlab::Ci::Config.new(config, opts) @config = @ci_config.to_hash unless @ci_config.valid? @@ -73,11 +73,11 @@ module Gitlab end end - def self.validation_message(content, project = nil, opts = {}) + def self.validation_message(content, opts = {}) return 'Please provide content of .gitlab-ci.yml' if content.blank? begin - Gitlab::Ci::YamlProcessor.new(content, project, opts) + Gitlab::Ci::YamlProcessor.new(content, opts) nil rescue ValidationError, ::Gitlab::Ci::ExternalFiles::Processor::ExternalFileError => e e.message -- cgit v1.2.1