diff options
20 files changed, 1116 insertions, 23 deletions
diff --git a/app/models/blob_viewer/gitlab_ci_yml.rb b/app/models/blob_viewer/gitlab_ci_yml.rb index 1a86f04b1b9..655241c2808 100644 --- a/app/models/blob_viewer/gitlab_ci_yml.rb +++ b/app/models/blob_viewer/gitlab_ci_yml.rb @@ -10,16 +10,16 @@ module BlobViewer self.file_types = %i(gitlab_ci) self.binary = false - def validation_message + def validation_message(project, sha) return @validation_message if defined?(@validation_message) prepare! - @validation_message = Gitlab::Ci::YamlProcessor.validation_message(blob.data) + @validation_message = Gitlab::Ci::YamlProcessor.validation_message(blob.data, { project: project, sha: sha }) end - def valid? - validation_message.blank? + def valid?(project, sha) + validation_message(project, sha).blank? end end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 2955e0b2bca..8c075253400 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -464,7 +464,7 @@ module Ci return @config_processor if defined?(@config_processor) @config_processor ||= begin - Gitlab::Ci::YamlProcessor.new(ci_yaml_file) + ::Gitlab::Ci::YamlProcessor.new(ci_yaml_file, { project: project, sha: sha }) rescue Gitlab::Ci::YamlProcessor::ValidationError => e self.yaml_errors = e.message nil diff --git a/app/models/repository.rb b/app/models/repository.rb index 99b2e5a6686..ad65881ff43 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -996,14 +996,6 @@ class Repository remote_branch: merge_request.target_branch) end - def blob_data_at(sha, path) - blob = blob_at(sha, path) - return unless blob - - blob.load_all_data! - blob.data - end - def squash(user, merge_request) raw.squash(user, merge_request.id, branch: merge_request.target_branch, start_sha: merge_request.diff_start_sha, @@ -1012,6 +1004,14 @@ class Repository message: merge_request.title) end + def blob_data_at(sha, path) + blob = blob_at(sha, path) + return unless blob + + blob.load_all_data! + blob.data + end + private # TODO Generice finder, later split this on finders by Ref or Oid diff --git a/app/views/projects/blob/viewers/_gitlab_ci_yml.html.haml b/app/views/projects/blob/viewers/_gitlab_ci_yml.html.haml index 28c5be6ebf3..5be7cc7f25a 100644 --- a/app/views/projects/blob/viewers/_gitlab_ci_yml.html.haml +++ b/app/views/projects/blob/viewers/_gitlab_ci_yml.html.haml @@ -1,9 +1,9 @@ -- if viewer.valid? +- if viewer.valid?(@project, @commit.sha) = icon('check fw') This GitLab CI configuration is valid. - else = icon('warning fw') This GitLab CI configuration is invalid: - = viewer.validation_message + = viewer.validation_message(@project, @commit.sha) = link_to 'Learn more', help_page_path('ci/yaml/README') diff --git a/changelogs/unreleased/42861-move-include-external-files-in-gitlab-ci-yml-from-starter-to-libre.yml b/changelogs/unreleased/42861-move-include-external-files-in-gitlab-ci-yml-from-starter-to-libre.yml new file mode 100644 index 00000000000..171779817c8 --- /dev/null +++ b/changelogs/unreleased/42861-move-include-external-files-in-gitlab-ci-yml-from-starter-to-libre.yml @@ -0,0 +1,5 @@ +--- +title: Move including external files in .gitlab-ci.yml from Starter to Libre +merge_request: 21603 +author: +type: changed diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 72b90ac6334..d069b94e53b 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -1352,6 +1352,187 @@ test: retry: 2 ``` +## `include` + +> Introduced in [GitLab Edition Premium][ee] 10.5. +> Available for Starter, Premium and Ultimate [versions][gitlab-versions] since 10.6. +> Behaviour expanded in GitLab 10.8 to allow more flexible overriding. +> Available for Libre since [11.4](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/21603) + +Using the `include` keyword, you can allow the inclusion of external YAML files. + +In the following example, the content of `.before-script-template.yml` will be +automatically fetched and evaluated along with the content of `.gitlab-ci.yml`: + +```yaml +# Content of https://gitlab.com/awesome-project/raw/master/.before-script-template.yml + +before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - gem install bundler --no-ri --no-rdoc + - bundle install --jobs $(nproc) "${FLAGS[@]}" +``` + +```yaml +# Content of .gitlab-ci.yml + +include: 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' + +rspec: + script: + - bundle exec rspec +``` + +You can define it either as a single string, or, in case you want to include +more than one files, an array of different values . The following examples +are both valid cases: + +```yaml +# Single string + +include: '/templates/.after-script-template.yml' +``` + +```yaml +# Array + +include: + - 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' + - '/templates/.after-script-template.yml' +``` + +--- + +`include` supports two types of files: + +- **local** to the same repository, referenced by using full paths in the same + repository, with `/` being the root directory. For example: + + ```yaml + # Within the repository + include: '/templates/.gitlab-ci-template.yml' + ``` + + NOTE: **Note:** + You can only use files that are currently tracked by Git on the same branch + your configuration file is. In other words, when using a **local file**, make + sure that both `.gitlab-ci.yml` and the local file are on the same branch. + + NOTE: **Note:** + We don't support the inclusion of local files through Git submodules paths. + +- **remote** in a different location, accessed using HTTP/HTTPS, referenced + using the full URL. For example: + + ```yaml + include: 'https://gitlab.com/awesome-project/raw/master/.gitlab-ci-template.yml' + ``` + + NOTE: **Note:** + The remote file must be publicly accessible through a simple GET request, as we don't support authentication schemas in the remote URL. + +--- + + +Since GitLab 10.8 we are now recursively merging the files defined in `include` +with those in `.gitlab-ci.yml`. Files defined by `include` are always +evaluated first and recursively merged with the content of `.gitlab-ci.yml`, no +matter the position of the `include` keyword. You can take advantage of +recursive merging to customize and override details in included CI +configurations with local definitions. + +The following example shows specific YAML-defined variables and details of the +`production` job from an include file being customized in `.gitlab-ci.yml`. + +```yaml +# Content of https://company.com/autodevops-template.yml + +variables: + POSTGRES_USER: user + POSTGRES_PASSWORD: testing_password + POSTGRES_DB: $CI_ENVIRONMENT_SLUG + +production: + stage: production + script: + - install_dependencies + - deploy + environment: + name: production + url: https://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + only: + - master +``` + +```yaml +# Content of .gitlab-ci.yml + +include: 'https://company.com/autodevops-template.yml' + +image: alpine:latest + +variables: + POSTGRES_USER: root + POSTGRES_PASSWORD: secure_password + +stages: + - build + - test + - production + +production: + environment: + url: https://domain.com +``` + +In this case, the variables `POSTGRES_USER` and `POSTGRES_PASSWORD` along +with the environment url of the `production` job defined in +`autodevops-template.yml` have been overridden by new values defined in +`.gitlab-ci.yml`. + +NOTE: **Note:** +Recursive includes are not supported meaning your external files +should not use the `include` keyword, as it will be ignored. + +Recursive merging lets you extend and override dictionary mappings, but +you cannot add or modify items to an included array. For example, to add +an additional item to the production job script, you must repeat the +existing script items. + +```yaml +# Content of https://company.com/autodevops-template.yml + +production: + stage: production + script: + - install_dependencies + - deploy +``` + +```yaml +# Content of .gitlab-ci.yml + +include: 'https://company.com/autodevops-template.yml' + +stages: + - production + +production: + script: + - install_depedencies + - deploy + - notify_owner +``` + +In this case, if `install_dependencies` and `deploy` were not repeated in +`.gitlab-ci.yml`, they would not be part of the script for the `production` +job in the combined CI configuration. + +NOTE: **Note:** +We currently do not support using YAML aliases across different YAML files +sourced by `include`. You must only refer to aliases in the same file. Instead +of using YAML anchors you can use [`extends` keyword](#extends). + ## `variables` > Introduced in GitLab Runner v0.5.0. diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 46dad59eb8c..fe98d25af29 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -1,6 +1,6 @@ module Gitlab module Ci - ## + # # Base GitLab CI Configuration facade # class Config @@ -15,6 +15,8 @@ module Gitlab @global.compose! rescue Loader::FormatError, Extendable::ExtensionError => e raise Config::ConfigError, e.message + rescue ::Gitlab::Ci::External::Processor::FileError => e + raise ::Gitlab::Ci::YamlProcessor::ValidationError, e.message end def valid? @@ -64,9 +66,22 @@ module Gitlab @global.jobs_value end - # 'opts' argument is used in EE see /ee/lib/ee/gitlab/ci/config.rb + private + def build_config(config, opts = {}) - Loader.new(config).load! + initial_config = Loader.new(config).load! + project = opts.fetch(:project, nil) + + if project + process_external_files(initial_config, project, opts) + else + initial_config + end + end + + def process_external_files(config, project, opts) + sha = opts.fetch(:sha) { project.repository.root_ref_sha } + ::Gitlab::Ci::External::Processor.new(config, project, sha).perform end end end diff --git a/lib/gitlab/ci/external/file/base.rb b/lib/gitlab/ci/external/file/base.rb new file mode 100644 index 00000000000..f4da07b0b02 --- /dev/null +++ b/lib/gitlab/ci/external/file/base.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + module File + class Base + YAML_WHITELIST_EXTENSION = /(yml|yaml)$/i.freeze + + def initialize(location, opts = {}) + @location = location + end + + def valid? + location.match(YAML_WHITELIST_EXTENSION) && content + end + + def content + raise NotImplementedError, 'content must be implemented and return a string or nil' + end + + def error_message + raise NotImplementedError, 'error_message must be implemented and return a string' + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/file/local.rb b/lib/gitlab/ci/external/file/local.rb new file mode 100644 index 00000000000..1aa7f687507 --- /dev/null +++ b/lib/gitlab/ci/external/file/local.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + module File + class Local < Base + attr_reader :location, :project, :sha + + def initialize(location, opts = {}) + super + + @project = opts.fetch(:project) + @sha = opts.fetch(:sha) + end + + def content + @content ||= fetch_local_content + end + + def error_message + "Local file '#{location}' is not valid." + end + + private + + def fetch_local_content + project.repository.blob_data_at(sha, location) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/file/remote.rb b/lib/gitlab/ci/external/file/remote.rb new file mode 100644 index 00000000000..59bb3e8999e --- /dev/null +++ b/lib/gitlab/ci/external/file/remote.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + module File + class Remote < Base + include Gitlab::Utils::StrongMemoize + attr_reader :location + + def content + return @content if defined?(@content) + + @content = strong_memoize(:content) do + begin + Gitlab::HTTP.get(location) + rescue Gitlab::HTTP::Error, Timeout::Error, SocketError, Gitlab::HTTP::BlockedUrlError + nil + end + end + end + + def error_message + "Remote file '#{location}' is not valid." + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/mapper.rb b/lib/gitlab/ci/external/mapper.rb new file mode 100644 index 00000000000..58bd6a19acf --- /dev/null +++ b/lib/gitlab/ci/external/mapper.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + class Mapper + def initialize(values, project, sha) + @locations = Array(values.fetch(:include, [])) + @project = project + @sha = sha + end + + def process + locations.map { |location| build_external_file(location) } + end + + private + + attr_reader :locations, :project, :sha + + def build_external_file(location) + if ::Gitlab::UrlSanitizer.valid?(location) + Gitlab::Ci::External::File::Remote.new(location) + else + options = { project: project, sha: sha } + Gitlab::Ci::External::File::Local.new(location, options) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/processor.rb b/lib/gitlab/ci/external/processor.rb new file mode 100644 index 00000000000..76cf3ce89f9 --- /dev/null +++ b/lib/gitlab/ci/external/processor.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + class Processor + FileError = Class.new(StandardError) + + def initialize(values, project, sha) + @values = values + @external_files = Gitlab::Ci::External::Mapper.new(values, project, sha).process + @content = {} + end + + def perform + return values if external_files.empty? + + external_files.each do |external_file| + validate_external_file(external_file) + @content.deep_merge!(content_of(external_file)) + end + + append_inline_content + remove_include_keyword + end + + private + + attr_reader :values, :external_files, :content + + def validate_external_file(external_file) + unless external_file.valid? + raise FileError, external_file.error_message + end + end + + def content_of(external_file) + Gitlab::Ci::Config::Loader.new(external_file.content).load! + end + + def append_inline_content + @content.deep_merge!(@values) + end + + def remove_include_keyword + content.delete(:include) + content + end + end + end + end +end diff --git a/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml b/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml new file mode 100644 index 00000000000..0bab94a7c2e --- /dev/null +++ b/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml @@ -0,0 +1,10 @@ +before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - ruby -v + - which ruby + - gem install bundler --no-ri --no-rdoc + - bundle install --jobs $(nproc) "${FLAGS[@]}" + +rspec: + script: + - bundle exec rspec diff --git a/spec/lib/gitlab/ci/config_spec.rb b/spec/lib/gitlab/ci/config_spec.rb index 5a78ce783dd..b43aca8a354 100644 --- a/spec/lib/gitlab/ci/config_spec.rb +++ b/spec/lib/gitlab/ci/config_spec.rb @@ -124,4 +124,237 @@ describe Gitlab::Ci::Config do end end end + + context "when using 'include' directive" do + let(:project) { create(:project, :repository) } + let(:remote_location) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + let(:local_location) { 'spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' } + + let(:remote_file_content) do + <<~HEREDOC + variables: + AUTO_DEVOPS_DOMAIN: domain.example.com + POSTGRES_USER: user + POSTGRES_PASSWORD: testing-password + POSTGRES_ENABLED: "true" + POSTGRES_DB: $CI_ENVIRONMENT_SLUG + HEREDOC + end + + let(:local_file_content) do + File.read(Rails.root.join(local_location)) + end + + let(:gitlab_ci_yml) do + <<~HEREDOC + include: + - #{local_location} + - #{remote_location} + + image: ruby:2.2 + HEREDOC + end + + let(:config) do + described_class.new(gitlab_ci_yml, project: project, sha: '12345') + end + + before do + WebMock.stub_request(:get, remote_location) + .to_return(body: remote_file_content) + + allow(project.repository) + .to receive(:blob_data_at).and_return(local_file_content) + end + + context "when gitlab_ci_yml has valid 'include' defined" do + it 'should return a composed hash' do + before_script_values = [ + "apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs", "ruby -v", + "which ruby", + "gem install bundler --no-ri --no-rdoc", + "bundle install --jobs $(nproc) \"${FLAGS[@]}\"" + ] + variables = { + AUTO_DEVOPS_DOMAIN: "domain.example.com", + POSTGRES_USER: "user", + POSTGRES_PASSWORD: "testing-password", + POSTGRES_ENABLED: "true", + POSTGRES_DB: "$CI_ENVIRONMENT_SLUG" + } + composed_hash = { + before_script: before_script_values, + image: "ruby:2.2", + rspec: { script: ["bundle exec rspec"] }, + variables: variables + } + + expect(config.to_hash).to eq(composed_hash) + end + end + + context "when gitlab_ci.yml has invalid 'include' defined" do + let(:gitlab_ci_yml) do + <<~HEREDOC + include: invalid + HEREDOC + end + + it 'raises error YamlProcessor validationError' do + expect { config }.to raise_error( + ::Gitlab::Ci::YamlProcessor::ValidationError, + "Local file 'invalid' is not valid." + ) + end + end + + describe 'external file version' do + context 'when external local file SHA is defined' do + it 'is using a defined value' do + expect(project.repository).to receive(:blob_data_at) + .with('eeff1122', local_location) + + described_class.new(gitlab_ci_yml, project: project, sha: 'eeff1122') + end + end + + context 'when external local file SHA is not defined' do + it 'is using latest SHA on the default branch' do + expect(project.repository).to receive(:root_ref_sha) + + described_class.new(gitlab_ci_yml, project: project) + end + end + end + + context "when both external files and gitlab_ci.yml defined the same key" do + let(:gitlab_ci_yml) do + <<~HEREDOC + include: + - #{remote_location} + + image: ruby:2.2 + HEREDOC + end + + let(:remote_file_content) do + <<~HEREDOC + image: php:5-fpm-alpine + HEREDOC + end + + it 'should take precedence' do + expect(config.to_hash).to eq({ image: 'ruby:2.2' }) + end + end + + context "when both external files and gitlab_ci.yml define a dictionary of distinct variables" do + let(:remote_file_content) do + <<~HEREDOC + variables: + A: 'alpha' + B: 'beta' + HEREDOC + end + + let(:gitlab_ci_yml) do + <<~HEREDOC + include: + - #{remote_location} + + variables: + C: 'gamma' + D: 'delta' + HEREDOC + end + + it 'should merge the variables dictionaries' do + expect(config.to_hash).to eq({ variables: { A: 'alpha', B: 'beta', C: 'gamma', D: 'delta' } }) + end + end + + context "when both external files and gitlab_ci.yml define a dictionary of overlapping variables" do + let(:remote_file_content) do + <<~HEREDOC + variables: + A: 'alpha' + B: 'beta' + C: 'omnicron' + HEREDOC + end + + let(:gitlab_ci_yml) do + <<~HEREDOC + include: + - #{remote_location} + + variables: + C: 'gamma' + D: 'delta' + HEREDOC + end + + it 'later declarations should take precedence' do + expect(config.to_hash).to eq({ variables: { A: 'alpha', B: 'beta', C: 'gamma', D: 'delta' } }) + end + end + + context 'when both external files and gitlab_ci.yml define a job' do + let(:remote_file_content) do + <<~HEREDOC + job1: + script: + - echo 'hello from remote file' + HEREDOC + end + + let(:gitlab_ci_yml) do + <<~HEREDOC + include: + - #{remote_location} + + job1: + variables: + VARIABLE_DEFINED_IN_MAIN_FILE: 'some value' + HEREDOC + end + + it 'merges the jobs' do + expect(config.to_hash).to eq({ + job1: { + script: ["echo 'hello from remote file'"], + variables: { + VARIABLE_DEFINED_IN_MAIN_FILE: 'some value' + } + } + }) + end + + context 'when the script key is in both' do + let(:gitlab_ci_yml) do + <<~HEREDOC + include: + - #{remote_location} + + job1: + script: + - echo 'hello from main file' + variables: + VARIABLE_DEFINED_IN_MAIN_FILE: 'some value' + HEREDOC + end + + it 'uses the script from the gitlab_ci.yml' do + expect(config.to_hash).to eq({ + job1: { + script: ["echo 'hello from main file'"], + variables: { + VARIABLE_DEFINED_IN_MAIN_FILE: 'some value' + } + } + }) + end + end + end + end end diff --git a/spec/lib/gitlab/ci/external/file/local_spec.rb b/spec/lib/gitlab/ci/external/file/local_spec.rb new file mode 100644 index 00000000000..3f32d81a827 --- /dev/null +++ b/spec/lib/gitlab/ci/external/file/local_spec.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Ci::External::File::Local do + let(:project) { create(:project, :repository) } + let(:local_file) { described_class.new(location, { project: project, sha: '12345' }) } + + describe '#valid?' do + context 'when is a valid local path' do + let(:location) { '/vendor/gitlab-ci-yml/existent-file.yml' } + + before do + allow_any_instance_of(described_class).to receive(:fetch_local_content).and_return("image: 'ruby2:2'") + end + + it 'should return true' do + expect(local_file.valid?).to be_truthy + end + end + + context 'when is not a valid local path' do + let(:location) { '/vendor/gitlab-ci-yml/non-existent-file.yml' } + + it 'should return false' do + expect(local_file.valid?).to be_falsy + end + end + + context 'when is not a yaml file' do + let(:location) { '/config/application.rb' } + + it 'should return false' do + expect(local_file.valid?).to be_falsy + end + end + end + + describe '#content' do + context 'with a a valid file' do + let(:local_file_content) do + <<~HEREDOC + before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - ruby -v + - which ruby + - gem install bundler --no-ri --no-rdoc + - bundle install --jobs $(nproc) "${FLAGS[@]}" + HEREDOC + end + let(:location) { '/vendor/gitlab-ci-yml/existent-file.yml' } + + before do + allow_any_instance_of(described_class).to receive(:fetch_local_content).and_return(local_file_content) + end + + it 'should return the content of the file' do + expect(local_file.content).to eq(local_file_content) + end + end + + context 'with an invalid file' do + let(:location) { '/vendor/gitlab-ci-yml/non-existent-file.yml' } + + it 'should be nil' do + expect(local_file.content).to be_nil + end + end + end + + describe '#error_message' do + let(:location) { '/vendor/gitlab-ci-yml/non-existent-file.yml' } + + it 'should return an error message' do + expect(local_file.error_message).to eq("Local file '#{location}' is not valid.") + end + end +end diff --git a/spec/lib/gitlab/ci/external/file/remote_spec.rb b/spec/lib/gitlab/ci/external/file/remote_spec.rb new file mode 100644 index 00000000000..b1819c8960b --- /dev/null +++ b/spec/lib/gitlab/ci/external/file/remote_spec.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Ci::External::File::Remote do + let(:remote_file) { described_class.new(location) } + let(:location) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + let(:remote_file_content) do + <<~HEREDOC + before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - ruby -v + - which ruby + - gem install bundler --no-ri --no-rdoc + - bundle install --jobs $(nproc) "${FLAGS[@]}" + HEREDOC + end + + describe "#valid?" do + context 'when is a valid remote url' do + before do + WebMock.stub_request(:get, location).to_return(body: remote_file_content) + end + + it 'should return true' do + expect(remote_file.valid?).to be_truthy + end + end + + context 'with an irregular url' do + let(:location) { 'not-valid://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + + it 'should return false' do + expect(remote_file.valid?).to be_falsy + end + end + + context 'with a timeout' do + before do + allow(Gitlab::HTTP).to receive(:get).and_raise(Timeout::Error) + end + + it 'should be falsy' do + expect(remote_file.valid?).to be_falsy + end + end + + context 'when is not a yaml file' do + let(:location) { 'https://asdasdasdaj48ggerexample.com' } + + it 'should be falsy' do + expect(remote_file.valid?).to be_falsy + end + end + + context 'with an internal url' do + let(:location) { 'http://localhost:8080' } + + it 'should be falsy' do + expect(remote_file.valid?).to be_falsy + end + end + end + + describe "#content" do + context 'with a valid remote file' do + before do + WebMock.stub_request(:get, location).to_return(body: remote_file_content) + end + + it 'should return the content of the file' do + expect(remote_file.content).to eql(remote_file_content) + end + end + + context 'with a timeout' do + before do + allow(Gitlab::HTTP).to receive(:get).and_raise(Timeout::Error) + end + + it 'should be falsy' do + expect(remote_file.content).to be_falsy + end + end + + context 'with an invalid remote url' do + let(:location) { 'https://asdasdasdaj48ggerexample.com' } + + before do + WebMock.stub_request(:get, location).to_raise(SocketError.new('Some HTTP error')) + end + + it 'should be nil' do + expect(remote_file.content).to be_nil + end + end + + context 'with an internal url' do + let(:location) { 'http://localhost:8080' } + + it 'should be nil' do + expect(remote_file.content).to be_nil + end + end + end + + describe "#error_message" do + let(:location) { 'not-valid://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + + it 'should return an error message' do + expect(remote_file.error_message).to eq("Remote file '#{location}' is not valid.") + end + end +end diff --git a/spec/lib/gitlab/ci/external/mapper_spec.rb b/spec/lib/gitlab/ci/external/mapper_spec.rb new file mode 100644 index 00000000000..6270d27a36d --- /dev/null +++ b/spec/lib/gitlab/ci/external/mapper_spec.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Ci::External::Mapper do + let(:project) { create(:project, :repository) } + let(:file_content) do + <<~HEREDOC + image: 'ruby:2.2' + HEREDOC + end + + describe '#process' do + subject { described_class.new(values, project, '123456').process } + + context "when 'include' keyword is defined as string" do + context 'when the string is a local file' do + let(:values) do + { + include: '/vendor/gitlab-ci-yml/non-existent-file.yml', + image: 'ruby:2.2' + } + end + + it 'returns an array' do + expect(subject).to be_an(Array) + end + + it 'returns File instances' do + expect(subject.first).to be_an_instance_of(Gitlab::Ci::External::File::Local) + end + end + + context 'when the string is a remote file' do + let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + let(:values) do + { + include: remote_url, + image: 'ruby:2.2' + } + end + + before do + WebMock.stub_request(:get, remote_url).to_return(body: file_content) + end + + it 'returns an array' do + expect(subject).to be_an(Array) + end + + it 'returns File instances' do + expect(subject.first).to be_an_instance_of(Gitlab::Ci::External::File::Remote) + end + end + end + + context "when 'include' is defined as an array" do + let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + let(:values) do + { + include: + [ + remote_url, + '/vendor/gitlab-ci-yml/template.yml' + ], + image: 'ruby:2.2' + } + end + + before do + WebMock.stub_request(:get, remote_url).to_return(body: file_content) + end + + it 'returns an array' do + expect(subject).to be_an(Array) + end + + it 'returns Files instances' do + expect(subject).to all(respond_to(:valid?)) + expect(subject).to all(respond_to(:content)) + end + end + + context "when 'include' is not defined" do + let(:values) do + { + image: 'ruby:2.2' + } + end + + it 'returns an empty array' do + expect(subject).to be_empty + end + end + end +end diff --git a/spec/lib/gitlab/ci/external/processor_spec.rb b/spec/lib/gitlab/ci/external/processor_spec.rb new file mode 100644 index 00000000000..688c2b3c8aa --- /dev/null +++ b/spec/lib/gitlab/ci/external/processor_spec.rb @@ -0,0 +1,182 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Ci::External::Processor do + let(:project) { create(:project, :repository) } + let(:processor) { described_class.new(values, project, '12345') } + + describe "#perform" do + context 'when no external files defined' do + let(:values) { { image: 'ruby:2.2' } } + + it 'should return the same values' do + expect(processor.perform).to eq(values) + end + end + + context 'when an invalid local file is defined' do + let(:values) { { include: '/vendor/gitlab-ci-yml/non-existent-file.yml', image: 'ruby:2.2' } } + + it 'should raise an error' do + expect { processor.perform }.to raise_error( + described_class::FileError, + "Local file '/vendor/gitlab-ci-yml/non-existent-file.yml' is not valid." + ) + end + end + + context 'when an invalid remote file is defined' do + let(:remote_file) { 'http://doesntexist.com/.gitlab-ci-1.yml' } + let(:values) { { include: remote_file, image: 'ruby:2.2' } } + + before do + WebMock.stub_request(:get, remote_file).to_raise(SocketError.new('Some HTTP error')) + end + + it 'should raise an error' do + expect { processor.perform }.to raise_error( + described_class::FileError, + "Remote file '#{remote_file}' is not valid." + ) + end + end + + context 'with a valid remote external file is defined' do + let(:remote_file) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + let(:values) { { include: remote_file, image: 'ruby:2.2' } } + let(:external_file_content) do + <<-HEREDOC + before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - ruby -v + - which ruby + - gem install bundler --no-ri --no-rdoc + - bundle install --jobs $(nproc) "${FLAGS[@]}" + + rspec: + script: + - bundle exec rspec + + rubocop: + script: + - bundle exec rubocop + HEREDOC + end + + before do + WebMock.stub_request(:get, remote_file).to_return(body: external_file_content) + end + + it 'should append the file to the values' do + output = processor.perform + expect(output.keys).to match_array([:image, :before_script, :rspec, :rubocop]) + end + + it "should remove the 'include' keyword" do + expect(processor.perform[:include]).to be_nil + end + end + + context 'with a valid local external file is defined' do + let(:values) { { include: '/vendor/gitlab-ci-yml/template.yml', image: 'ruby:2.2' } } + let(:local_file_content) do + <<-HEREDOC + before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - ruby -v + - which ruby + - gem install bundler --no-ri --no-rdoc + - bundle install --jobs $(nproc) "${FLAGS[@]}" + HEREDOC + end + + before do + allow_any_instance_of(Gitlab::Ci::External::File::Local).to receive(:fetch_local_content).and_return(local_file_content) + end + + it 'should append the file to the values' do + output = processor.perform + expect(output.keys).to match_array([:image, :before_script]) + end + + it "should remove the 'include' keyword" do + expect(processor.perform[:include]).to be_nil + end + end + + context 'with multiple external files are defined' do + let(:remote_file) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + let(:external_files) do + [ + '/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml', + remote_file + ] + end + let(:values) do + { + include: external_files, + image: 'ruby:2.2' + } + end + + let(:remote_file_content) do + <<-HEREDOC + stages: + - build + - review + - cleanup + HEREDOC + end + + before do + local_file_content = File.read(Rails.root.join('spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml')) + allow_any_instance_of(Gitlab::Ci::External::File::Local).to receive(:fetch_local_content).and_return(local_file_content) + WebMock.stub_request(:get, remote_file).to_return(body: remote_file_content) + end + + it 'should append the files to the values' do + expect(processor.perform.keys).to match_array([:image, :stages, :before_script, :rspec]) + end + + it "should remove the 'include' keyword" do + expect(processor.perform[:include]).to be_nil + end + end + + context 'when external files are defined but not valid' do + let(:values) { { include: '/vendor/gitlab-ci-yml/template.yml', image: 'ruby:2.2' } } + + let(:local_file_content) { 'invalid content file ////' } + + before do + allow_any_instance_of(Gitlab::Ci::External::File::Local).to receive(:fetch_local_content).and_return(local_file_content) + end + + it 'should raise an error' do + expect { processor.perform }.to raise_error(Gitlab::Ci::Config::Loader::FormatError) + end + end + + context "when both external files and values defined the same key" do + let(:remote_file) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + let(:values) do + { + include: remote_file, + image: 'ruby:2.2' + } + end + + let(:remote_file_content) do + <<~HEREDOC + image: php:5-fpm-alpine + HEREDOC + end + + it 'should take precedence' do + WebMock.stub_request(:get, remote_file).to_return(body: remote_file_content) + expect(processor.perform[:image]).to eq('ruby:2.2') + end + end + end +end diff --git a/spec/models/blob_viewer/gitlab_ci_yml_spec.rb b/spec/models/blob_viewer/gitlab_ci_yml_spec.rb index bed364a8c14..01c555a7a90 100644 --- a/spec/models/blob_viewer/gitlab_ci_yml_spec.rb +++ b/spec/models/blob_viewer/gitlab_ci_yml_spec.rb @@ -2,22 +2,24 @@ require 'spec_helper' describe BlobViewer::GitlabCiYml do include FakeBlobHelpers + include RepoHelpers - let(:project) { build_stubbed(:project) } + let(:project) { create(:project, :repository) } let(:data) { File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml')) } let(:blob) { fake_blob(path: '.gitlab-ci.yml', data: data) } + let(:sha) { sample_commit.id } subject { described_class.new(blob) } describe '#validation_message' do it 'calls prepare! on the viewer' do expect(subject).to receive(:prepare!) - subject.validation_message + subject.validation_message(project, sha) end context 'when the configuration is valid' do it 'returns nil' do - expect(subject.validation_message).to be_nil + expect(subject.validation_message(project, sha)).to be_nil end end @@ -25,7 +27,7 @@ describe BlobViewer::GitlabCiYml do let(:data) { 'oof' } it 'returns the error message' do - expect(subject.validation_message).to eq('Invalid configuration format') + expect(subject.validation_message(project, sha)).to eq('Invalid configuration format') end end end diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index 14ccc2960bb..2216705c032 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -1743,7 +1743,7 @@ describe Ci::Pipeline, :mailer do create(:ci_pipeline, config: { rspec: { script: 'rake test' } }) end - it 'does not containyaml errors' do + it 'does not contain yaml errors' do expect(pipeline).not_to have_yaml_errors end end |