diff options
| -rw-r--r-- | lib/gitlab/ci/config/entry/global.rb | 10 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/entry/includes.rb | 22 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/entry/validators.rb | 32 | ||||
| -rw-r--r-- | lib/gitlab/ci/external_files/external_file.rb | 38 | ||||
| -rw-r--r-- | lib/gitlab/ci/external_files/mapper.rb | 27 | ||||
| -rw-r--r-- | lib/gitlab/ci/external_files/processor.rb | 46 | ||||
| -rw-r--r-- | spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml | 10 | ||||
| -rw-r--r-- | spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-2.yml | 8 | ||||
| -rw-r--r-- | spec/lib/gitlab/ci/config/entry/global_spec.rb | 6 | ||||
| -rw-r--r-- | spec/lib/gitlab/ci/config/entry/includes_spec.rb | 97 | ||||
| -rw-r--r-- | spec/lib/gitlab/ci/external_files/external_file_spec.rb | 78 | ||||
| -rw-r--r-- | spec/lib/gitlab/ci/external_files/mapper_spec.rb | 39 | ||||
| -rw-r--r-- | spec/lib/gitlab/ci/external_files/processor_spec.rb | 141 |
13 files changed, 550 insertions, 4 deletions
diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb index a4ec8f0ff2f..401e566de77 100644 --- a/lib/gitlab/ci/config/entry/global.rb +++ b/lib/gitlab/ci/config/entry/global.rb @@ -33,11 +33,15 @@ module Gitlab entry :cache, Entry::Cache, description: 'Configure caching between build jobs.' + entry :includes, Entry::Includes, + description: 'External GitlLab Ci files' + helpers :before_script, :image, :services, :after_script, - :variables, :stages, :types, :cache, :jobs + :variables, :stages, :types, :cache, :jobs, :includes def compose!(_deps = nil) super(self) do + append_external_files! compose_jobs! compose_deprecated_entries! end @@ -45,6 +49,10 @@ module Gitlab private + def append_external_files! + return if includes_value.nil? + end + def compose_jobs! factory = Entry::Factory.new(Entry::Jobs) .value(@config.except(*self.class.nodes.keys)) diff --git a/lib/gitlab/ci/config/entry/includes.rb b/lib/gitlab/ci/config/entry/includes.rb new file mode 100644 index 00000000000..c7a8c7960e8 --- /dev/null +++ b/lib/gitlab/ci/config/entry/includes.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a Docker image. + # + class Includes < Node + include Validatable + + validations do + validates :config, array_or_string: true, external_file: true, allow_nil: true + end + + def value + Array(@config) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index b3c889ee92f..5b4d5d53821 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -60,6 +60,38 @@ module Gitlab end end + class ArrayOrStringValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(Array) || value.is_a?(String) + record.errors.add(attribute, 'should be an array or a string') + end + end + end + + class ExternalFileValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + if value.is_a?(Array) + value.each do |path| + validate_external_file(path, record, attribute) + end + else + validate_external_file(value, record, attribute) + end + end + + private + + def validate_external_file(value, record, attribute) + unless valid_url?(value) + record.errors.add(attribute, 'should be a valid local or remote file') + end + end + + def valid_url?(value) + Gitlab::UrlSanitizer.valid?(value) || File.exists?("#{Rails.root}/#{value}") + end + end + class KeyValidator < ActiveModel::EachValidator include LegacyValidationHelpers diff --git a/lib/gitlab/ci/external_files/external_file.rb b/lib/gitlab/ci/external_files/external_file.rb new file mode 100644 index 00000000000..9bad8231c42 --- /dev/null +++ b/lib/gitlab/ci/external_files/external_file.rb @@ -0,0 +1,38 @@ +require 'open-uri' + +module Gitlab + module Ci + module ExternalFiles + class ExternalFile + + def initialize(value) + @value = value + end + + def content + if remote_url? + open(value).read + else + File.read(base_path) + end + end + + def valid? + remote_url? || File.exists?(base_path) + end + + private + + attr_reader :value + + def base_path + "#{Rails.root}/#{value}" + end + + def remote_url? + ::Gitlab::UrlSanitizer.valid?(value) + end + end + end + end +end diff --git a/lib/gitlab/ci/external_files/mapper.rb b/lib/gitlab/ci/external_files/mapper.rb new file mode 100644 index 00000000000..5deb2f5a2e5 --- /dev/null +++ b/lib/gitlab/ci/external_files/mapper.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module ExternalFiles + class Mapper + + def self.fetch_paths(values) + paths = values.fetch(:includes, []) + normalize_paths(paths) + end + + private + + def self.normalize_paths(paths) + if paths.is_a?(String) + [build_external_file(paths)] + else + paths.map { |path| build_external_file(path) } + end + end + + def self.build_external_file(path) + ::Gitlab::Ci::ExternalFiles::ExternalFile.new(path) + end + end + end + end +end diff --git a/lib/gitlab/ci/external_files/processor.rb b/lib/gitlab/ci/external_files/processor.rb new file mode 100644 index 00000000000..f38bc7633b5 --- /dev/null +++ b/lib/gitlab/ci/external_files/processor.rb @@ -0,0 +1,46 @@ +module Gitlab + module Ci + module ExternalFiles + class Processor + ExternalFileError = Class.new(StandardError) + + def initialize(values) + @values = values + @external_files = ::Gitlab::Ci::ExternalFiles::Mapper.fetch_paths(values) + end + + def perform + return values if external_files.empty? + + external_files.each do |external_file| + validate_external_file(external_file) + append_external_content(external_file) + end + + remove_include_keyword + end + + private + + attr_reader :values, :external_files + + def validate_external_file(external_file) + unless external_file.valid? + raise ExternalFileError, 'External files should be a valid local or remote file' + end + end + + def append_external_content(external_file) + external_values = ::Gitlab::Ci::Config::Loader.new(external_file.content).load! + @values.merge!(external_values) + end + + def remove_include_keyword + values.delete(:includes) + values + end + + end + end + end +end diff --git a/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml b/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml new file mode 100644 index 00000000000..0bab94a7c2e --- /dev/null +++ b/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml @@ -0,0 +1,10 @@ +before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - ruby -v + - which ruby + - gem install bundler --no-ri --no-rdoc + - bundle install --jobs $(nproc) "${FLAGS[@]}" + +rspec: + script: + - bundle exec rspec diff --git a/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-2.yml b/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-2.yml new file mode 100644 index 00000000000..b341cca8946 --- /dev/null +++ b/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-2.yml @@ -0,0 +1,8 @@ +variables: + # AUTO_DEVOPS_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. + + AUTO_DEVOPS_DOMAIN: domain.example.com + POSTGRES_USER: user + POSTGRES_PASSWORD: testing-password + POSTGRES_ENABLED: "true" + POSTGRES_DB: $CI_ENVIRONMENT_SLUG diff --git a/spec/lib/gitlab/ci/config/entry/global_spec.rb b/spec/lib/gitlab/ci/config/entry/global_spec.rb index 1860ed79bfd..ff623b95be8 100644 --- a/spec/lib/gitlab/ci/config/entry/global_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/global_spec.rb @@ -13,7 +13,7 @@ describe Gitlab::Ci::Config::Entry::Global do expect(described_class.nodes.keys) .to match_array(%i[before_script image services after_script variables stages - types cache]) + types cache includes]) end end end @@ -42,7 +42,7 @@ describe Gitlab::Ci::Config::Entry::Global do end it 'creates node object for each entry' do - expect(global.descendants.count).to eq 8 + expect(global.descendants.count).to eq 9 end it 'creates node object using valid class' do @@ -189,7 +189,7 @@ describe Gitlab::Ci::Config::Entry::Global do describe '#nodes' do it 'instantizes all nodes' do - expect(global.descendants.count).to eq 8 + expect(global.descendants.count).to eq 9 end it 'contains unspecified nodes' do diff --git a/spec/lib/gitlab/ci/config/entry/includes_spec.rb b/spec/lib/gitlab/ci/config/entry/includes_spec.rb new file mode 100644 index 00000000000..d72503535ed --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/includes_spec.rb @@ -0,0 +1,97 @@ +require 'rails_helper' + +describe Gitlab::Ci::Config::Entry::Includes do + let(:entry) { described_class.new(config) } + + shared_examples 'valid external file' do + it 'should be valid' do + expect(entry).to be_valid + end + + it 'should not return any error' do + expect(entry.errors).to be_empty + end + end + + shared_examples 'invalid external file' do + it 'should not be valid' do + expect(entry).not_to be_valid + end + + it 'should return an error' do + expect(entry.errors.first).to match(/should be a valid local or remote file/) + end + end + + describe "#valid?" do + context 'with no external file given' do + let(:config) { nil } + + it_behaves_like 'valid external file' + end + + context 'with multiple external files' do + let(:config) { %w(https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-2.yml) } + + it_behaves_like 'valid external file' + end + + context 'with just one external file' do + let(:config) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + + it_behaves_like 'valid external file' + end + + context 'when they contain valid URLs' do + let(:config) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + + it_behaves_like 'valid external file' + end + + context 'when they contain valid relative URLs' do + let(:config) { '/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml' } + + it_behaves_like 'valid external file' + end + + context 'when they not contain valid URLs' do + let(:config) { 'not-valid://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + + it_behaves_like 'invalid external file' + end + + context 'when they not contain valid relative URLs' do + let(:config) { '/vendor/gitlab-ci-yml/non-existent-file.yml' } + + it_behaves_like 'invalid external file' + end + end + + describe "#value" do + context 'with multiple external files' do + let(:config) { %w(https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-2.yml) } + it 'should return an array' do + expect(entry.value).to be_an(Array) + expect(entry.value.count).to eq(2) + end + end + + context 'with just one external file' do + let(:config) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + + it 'should return an array' do + expect(entry.value).to be_an(Array) + expect(entry.value.count).to eq(1) + end + end + + context 'with no external file given' do + let(:config) { nil } + + it 'should return an empty array' do + expect(entry.value).to be_an(Array) + expect(entry.value).to be_empty + end + end + end +end diff --git a/spec/lib/gitlab/ci/external_files/external_file_spec.rb b/spec/lib/gitlab/ci/external_files/external_file_spec.rb new file mode 100644 index 00000000000..80468f63435 --- /dev/null +++ b/spec/lib/gitlab/ci/external_files/external_file_spec.rb @@ -0,0 +1,78 @@ +require 'rails_helper' + +describe Gitlab::Ci::ExternalFiles::ExternalFile do + let(:external_file) { described_class.new(value) } + + describe "#valid?" do + context 'when is a valid remote url' do + let(:value) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + + it 'should return true' do + expect(external_file.valid?).to be_truthy + end + end + + context 'when is not a valid remote url' do + let(:value) { 'not-valid://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + + it 'should return false' do + expect(external_file.valid?).to be_falsy + end + end + + context 'when is a valid local path' do + let(:value) { '/vendor/gitlab-ci-yml/existent-file.yml' } + + it 'should return true' do + allow(File).to receive(:exists?).and_return(true) + expect(external_file.valid?).to be_truthy + end + end + + context 'when is not a valid local path' do + let(:value) { '/vendor/gitlab-ci-yml/non-existent-file.yml' } + + it 'should return false' do + expect(external_file.valid?).to be_falsy + end + end + end + + describe "#content" do + let(:external_file_content) { + <<-HEREDOC + before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - ruby -v + - which ruby + - gem install bundler --no-ri --no-rdoc + - bundle install --jobs $(nproc) "${FLAGS[@]}" + HEREDOC + } + + context 'with a local file' do + let(:value) { '/vendor/gitlab-ci-yml/non-existent-file.yml' } + + before do + allow(File).to receive(:exists?).and_return(true) + allow(File).to receive(:read).and_return(external_file_content) + end + + it 'should return the content of the file' do + expect(external_file.content).to eq(external_file_content) + end + end + + context 'with a valid remote file' do + let(:value) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + + before do + allow_any_instance_of(Kernel).to receive_message_chain(:open, :read).and_return(external_file_content) + end + + it 'should return the content of the file' do + expect(external_file.content).to eq(external_file_content) + end + end + end +end diff --git a/spec/lib/gitlab/ci/external_files/mapper_spec.rb b/spec/lib/gitlab/ci/external_files/mapper_spec.rb new file mode 100644 index 00000000000..dfbf8bfa098 --- /dev/null +++ b/spec/lib/gitlab/ci/external_files/mapper_spec.rb @@ -0,0 +1,39 @@ +require 'rails_helper' + +describe Gitlab::Ci::ExternalFiles::Mapper do + describe '.fetch_paths' do + context 'when includes is defined as string' do + let(:values) { { includes: '/vendor/gitlab-ci-yml/non-existent-file.yml', image: 'ruby:2.2'} } + + it 'returns an array' do + expect(described_class.fetch_paths(values)).to be_an(Array) + end + + it 'returns ExternalFile instances' do + expect(described_class.fetch_paths(values).first).to be_an_instance_of(::Gitlab::Ci::ExternalFiles::ExternalFile) + end + end + + context 'when includes is defined as an array' do + let(:values) { { includes: ['https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml', '/vendor/gitlab-ci-yml/template.yml'], image: 'ruby:2.2'} } + it 'returns an array' do + expect(described_class.fetch_paths(values)).to be_an(Array) + end + + it 'returns ExternalFile instances' do + paths = described_class.fetch_paths(values) + paths.each do |path| + expect(path).to be_an_instance_of(::Gitlab::Ci::ExternalFiles::ExternalFile) + end + end + end + + context 'when includes is not defined' do + let(:values) { { image: 'ruby:2.2'} } + + it 'returns an empty array' do + expect(described_class.fetch_paths(values)).to be_empty + end + end + end +end diff --git a/spec/lib/gitlab/ci/external_files/processor_spec.rb b/spec/lib/gitlab/ci/external_files/processor_spec.rb new file mode 100644 index 00000000000..51ed14de4cf --- /dev/null +++ b/spec/lib/gitlab/ci/external_files/processor_spec.rb @@ -0,0 +1,141 @@ +require 'rails_helper' + +describe Gitlab::Ci::ExternalFiles::Processor do + let(:processor) { described_class.new(values) } + + describe "#perform" do + context 'when no external files defined' do + let(:values) { { image: 'ruby:2.2' } } + + it 'should return the same values' do + expect(processor.perform).to eq(values) + end + end + + context 'when an invalid local file is defined' do + let(:values) { { includes: '/vendor/gitlab-ci-yml/non-existent-file.yml', image: 'ruby:2.2'} } + + it 'should raise an error' do + expect { processor.perform }.to raise_error(described_class::ExternalFileError) + end + end + + context 'when an invalid remote file is defined' do + let(:values) { { includes: 'not-valid://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml', image: 'ruby:2.2'} } + + it 'should raise an error' do + expect { processor.perform }.to raise_error(described_class::ExternalFileError) + end + end + + context 'with a valid remote external file is defined' do + let(:values) { { includes: 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml', image: 'ruby:2.2' } } + let(:external_file_content) { + <<-HEREDOC + before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - ruby -v + - which ruby + - gem install bundler --no-ri --no-rdoc + - bundle install --jobs $(nproc) "${FLAGS[@]}" + + rspec: + script: + - bundle exec rspec + + rubocop: + script: + - bundle exec rubocop + HEREDOC + } + + before do + allow_any_instance_of(Kernel).to receive_message_chain(:open, :read).and_return(external_file_content) + end + + it 'should append the file to the values' do + output = processor.perform + expect(output.keys).to match_array([:image, :before_script, :rspec, :rubocop]) + end + + it "should remove the 'includes' keyword" do + expect(processor.perform[:includes]).to be_nil + end + end + + context 'with a valid local external file is defined' do + let(:values) { { includes: '/vendor/gitlab-ci-yml/template.yml' , image: 'ruby:2.2'} } + let(:external_file_content) { + <<-HEREDOC + before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - ruby -v + - which ruby + - gem install bundler --no-ri --no-rdoc + - bundle install --jobs $(nproc) "${FLAGS[@]}" + HEREDOC + } + + before do + allow(File).to receive(:exists?).and_return(true) + allow(File).to receive(:read).and_return(external_file_content) + end + + it 'should append the file to the values' do + output = processor.perform + expect(output.keys).to match_array([:image, :before_script]) + end + + it "should remove the 'includes' keyword" do + expect(processor.perform[:includes]).to be_nil + end + end + + context 'with multiple external files are defined' do + let(:external_files) { + [ + "/spec/ee/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml", + "/spec/ee/fixtures/gitlab/ci/external_files/.gitlab-ci-template-2.yml", + 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' + ] + } + let(:values) { { includes: external_files, image: 'ruby:2.2'} } + + let(:remote_file_content) { + <<-HEREDOC + stages: + - build + - review + - cleanup + HEREDOC + } + + before do + allow_any_instance_of(Kernel).to receive_message_chain(:open, :read).and_return(remote_file_content) + end + + it 'should append the files to the values' do + expect(processor.perform.keys).to match_array([:image, :variables, :stages, :before_script, :rspec]) + end + + it "should remove the 'includes' keyword" do + expect(processor.perform[:includes]).to be_nil + end + end + + context 'when external files are defined but not valid' do + let(:values) { { includes: '/vendor/gitlab-ci-yml/template.yml', image: 'ruby:2.2'} } + + let(:external_file_content) { 'invalid content file ////' } + + before do + allow(File).to receive(:exists?).and_return(true) + allow(File).to receive(:read).and_return(external_file_content) + end + + it 'should raise an error' do + expect { processor.perform }.to raise_error(Gitlab::Ci::Config::Loader::FormatError) + end + end + end +end |