diff options
author | Kamil Trzciński <ayufan@ayufan.eu> | 2015-08-05 15:11:47 +0000 |
---|---|---|
committer | Kamil Trzciński <ayufan@ayufan.eu> | 2015-08-05 15:11:47 +0000 |
commit | c790a1d4a19c5b78fb4fe47f1466c842006456c0 (patch) | |
tree | 81fb41ab59f5581ce7341f4bb42cb78dbdacb119 | |
parent | 3ee72501882a48b89f14a58f0c6093d35c7bf9cd (diff) | |
parent | faee0750d08787993727f10a008c70aa63f8814c (diff) | |
download | gitlab-ci-c790a1d4a19c5b78fb4fe47f1466c842006456c0.tar.gz |
Merge branch 'yaml-variables' into 'master'
Support yaml variables
This MR introduces ability to define variables from YAML.
```yaml
variables:
DB_NAME: postgres
test:
script: echo $DB_NAME
```
The variables are passed using the same API as Secure Variables. The API introduces additional parameter: public. All variables defined in YAML are marked as public. The GitLab Runner when detects public variables will pass them to the services. This makes it easy to fine tune linked services to for example define database name.
```yaml
services:
- postgres
variables:
POSTGRES_DB: gitlab
```
The above example will run [postgres](https://registry.hub.docker.com/u/library/postgres/) and pass POSTGRES_DB to postgres container making it to create `gitlab` database instead of default `postges`.
**Note:** All variables will passed to all service containers. It's not designed to distinguish which variable should go where.
/cc @sytses @vsizov @dzaporozhets
See merge request !227
-rw-r--r-- | CHANGELOG | 1 | ||||
-rw-r--r-- | app/models/build.rb | 20 | ||||
-rw-r--r-- | doc/docker/using_docker_images.md | 33 | ||||
-rw-r--r-- | doc/variables/README.md | 32 | ||||
-rw-r--r-- | doc/yaml/README.md | 16 | ||||
-rw-r--r-- | lib/api/entities.rb | 6 | ||||
-rw-r--r-- | lib/gitlab_ci_yaml_processor.rb | 14 | ||||
-rw-r--r-- | spec/lib/gitlab_ci_yaml_processor_spec.rb | 33 | ||||
-rw-r--r-- | spec/requests/api/builds_spec.rb | 5 | ||||
-rw-r--r-- | spec/support/gitlab_stubs/gitlab_ci.yml | 3 |
10 files changed, 151 insertions, 12 deletions
@@ -8,6 +8,7 @@ v7.14.0 (unreleased) - Refactor GitLab API usage to use either access_token or private_token depending on what was specified during login - Allow to use access_token for API requests - Fix project API listing returning empty list when first projects are not added to CI + - Allow to define variables from YAML - Added support for CI skipped status diff --git a/app/models/build.rb b/app/models/build.rb index 11bd967..4f6bf32 100644 --- a/app/models/build.rb +++ b/app/models/build.rb @@ -164,7 +164,7 @@ class Build < ActiveRecord::Base end def variables - project.variables + yaml_variables + project_variables end def duration @@ -245,4 +245,22 @@ class Build < ActiveRecord::Base def path_to_trace "#{dir_to_trace}/#{id}.log" end + + private + + def yaml_variables + if commit.config_processor + commit.config_processor.variables.map do |key, value| + { key: key, value: value, public: true } + end + else + [] + end + end + + def project_variables + project.variables.map do |variable| + { key: variable.key, value: variable.value, public: false } + end + end end diff --git a/doc/docker/using_docker_images.md b/doc/docker/using_docker_images.md index a88bb62..ef449cd 100644 --- a/doc/docker/using_docker_images.md +++ b/doc/docker/using_docker_images.md @@ -66,6 +66,39 @@ Alias hostname for the service is made from the image name: 1. Everything after `:` is stripped, 2. '/' is replaced to `_`. +### Configuring services +Many services accept environment variables, which allow you to easily change database names or set account names depending on the environment. + +GitLab Runner 0.5.0 and up passes all YAML-defined variables to created service containers. + +1. To configure database name for [postgres](https://registry.hub.docker.com/u/library/postgres/) service, +you need to set POSTGRES_DB. + + ```yaml + services: + - postgres + + variables: + POSTGRES_DB: gitlab + ``` + +1. To use [mysql](https://registry.hub.docker.com/u/library/mysql/) service with empty password for time of build, +you need to set MYSQL_ALLOW_EMPTY_PASSWORD. + + ```yaml + services: + - mysql + + variables: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + ``` + +For other possible configuration variables check the +https://registry.hub.docker.com/u/library/mysql/ or https://registry.hub.docker.com/u/library/postgres/ +or README page for any other Docker image. + +**Note: All variables will passed to all service containers. It's not designed to distinguish which variable should go where.** + ### Overwrite image and services It's possible to overwrite `docker-image` and specify services from `.gitlab-ci.yml`. If you add to your YAML the `image` and the `services` these parameters diff --git a/doc/variables/README.md b/doc/variables/README.md index d61474a..04c6bf1 100644 --- a/doc/variables/README.md +++ b/doc/variables/README.md @@ -1,6 +1,17 @@ ## Variables When receiving a build from GitLab CI, the runner prepares the build environment. -It starts by setting a list of **predefined variables** (Environment Variables) and a list of **user-defined variables** (Secure Variables) +It starts by setting a list of **predefined variables** (Environment Variables) and a list of **user-defined variables** + +The variables can be overwritten. They take precedence over each other in this order: +1. Secure variables +1. YAML-defined variables +1. Predefined variables + +For example, if you define: +1. API_TOKEN=SECURE as Secure Variable +1. API_TOKEN=YAML as YAML-defined variable + +The API_TOKEN will take the Secure Variable value: `SECURE`. ### Predefined variables (Environment Variables) @@ -36,8 +47,25 @@ export CI_SERVER_REVISION="" export CI_SERVER_VERSION="" ``` +### YAML-defined variables +**This feature requires GitLab Runner 0.5.0 or higher** + +GitLab CI allows you to add to `.gitlab-ci.yml` variables that are set in build environment. +The variables are stored in repository and are meant to store non-sensitive project configuration, ie. RAILS_ENV or DATABASE_URL. + +```yaml +variables: + DATABASE_URL: "postgres://postgres@postgres/my_database" +``` + +These variables can be later used in all executed commands and scripts. + +The YAML-defined variables are also set to all created service containers, thus allowing to fine tune them. + +More information about Docker integration can be found in [Using Docker Images](../docker/using_docker_images.md). + ### User-defined variables (Secure Variables) -**This feature requires `gitlab-runner` with version equal or greater than 0.4.0.** +**This feature requires GitLab Runner 0.4.0 or higher** GitLab CI allows you to define per-project **Secure Variables** that are set in build environment. The secure variables are stored out of the repository (the `.gitlab-ci.yml`). diff --git a/doc/yaml/README.md b/doc/yaml/README.md index 301144a..4caecca 100644 --- a/doc/yaml/README.md +++ b/doc/yaml/README.md @@ -55,6 +55,7 @@ There are a few `keywords` that can't be used as job names: | stages | optional | Define build stages | | types | optional | Alias for `stages` | | before_script | optional | Define commands prepended for each job's script | +| variables | optional | Define build variables | ### image and services This allows to specify a custom Docker image and a list of services that can be used for time of the build. @@ -94,6 +95,21 @@ There are also two edge cases worth mentioning: ### types Alias for [stages](#stages). +### variables +**This feature requires `gitlab-runner` with version equal or greater than 0.5.0.** + +GitLab CI allows you to add to `.gitlab-ci.yml` variables that are set in build environment. +The variables are stored in repository and are meant to store non-sensitive project configuration, ie. RAILS_ENV or DATABASE_URL. + +```yaml +variables: + DATABASE_URL: "postgres://postgres@postgres/my_database" +``` + +These variables can be later used in all executed commands and scripts. + +The YAML-defined variables are also set to all created service containers, thus allowing to fine tune them. + ## Jobs `.gitlab-ci.yml` allows you to specify an unlimited number of jobs. Each job has to have a unique `job_name`, which is not one of the keywords mentioned above. diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 910db52..464667f 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -7,15 +7,11 @@ module API expose :builds end - class Variable < Grape::Entity - expose :key, :value - end - class Build < Grape::Entity expose :id, :commands, :path, :ref, :sha, :project_id, :repo_url, :before_sha, :timeout, :allow_git_fetch, :project_name, :options - expose :variables, using: Variable + expose :variables end class Runner < Grape::Entity diff --git a/lib/gitlab_ci_yaml_processor.rb b/lib/gitlab_ci_yaml_processor.rb index b88b682..e1da311 100644 --- a/lib/gitlab_ci_yaml_processor.rb +++ b/lib/gitlab_ci_yaml_processor.rb @@ -3,9 +3,10 @@ class GitlabCiYamlProcessor DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' + ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables] ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage] - attr_reader :before_script, :image, :services + attr_reader :before_script, :image, :services, :variables def initialize(config) @config = YAML.load(config) @@ -42,7 +43,8 @@ class GitlabCiYamlProcessor @image = @config[:image] @services = @config[:services] @stages = @config[:stages] || @config[:types] - @config.except!(:before_script, :image, :services, :types, :stages) + @variables = @config[:variables] || {} + @config.except!(*ALLOWED_YAML_KEYS) @config.each do |name, param| raise ValidationError, "Unknown parameter: #{name}" unless param.is_a?(Hash) @@ -128,6 +130,10 @@ class GitlabCiYamlProcessor raise ValidationError, "stages should be an array of strings" end + unless @variables.nil? || validate_variables(@variables) + raise ValidationError, "variables should be a map of key-valued strings" + end + @jobs.each do |name, job| validate_job!("#{name} job", job) end @@ -178,4 +184,8 @@ class GitlabCiYamlProcessor def validate_array_of_strings(values) values.is_a?(Array) && values.all? {|tag| tag.is_a?(String)} end + + def validate_variables(variables) + variables.is_a?(Hash) && variables.all? {|key, value| key.is_a?(Symbol) && value.is_a?(String)} + end end diff --git a/spec/lib/gitlab_ci_yaml_processor_spec.rb b/spec/lib/gitlab_ci_yaml_processor_spec.rb index fdd77ed..5b9a4f1 100644 --- a/spec/lib/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/gitlab_ci_yaml_processor_spec.rb @@ -153,6 +153,23 @@ describe GitlabCiYamlProcessor do end end + describe "Variables" do + it "returns variables when defined" do + variables = { + var1: "value1", + var2: "value2", + } + config = YAML.dump({ + variables: variables, + before_script: ["pwd"], + rspec: {script: "rspec"} + }) + + config_processor = GitlabCiYamlProcessor.new(config) + config_processor.variables.should == variables + end + end + describe "Error handling" do it "indicates that object is invalid" do expect{GitlabCiYamlProcessor.new("invalid_yaml\n!ccdvlf%612334@@@@")}.to raise_error(GitlabCiYamlProcessor::ValidationError) @@ -269,5 +286,19 @@ describe GitlabCiYamlProcessor do GitlabCiYamlProcessor.new(config) end.to raise_error(GitlabCiYamlProcessor::ValidationError, "stages should be an array of strings") end + + it "returns errors if variables is not a map" do + config = YAML.dump({variables: "test", rspec: {script: "test"}}) + expect do + GitlabCiYamlProcessor.new(config) + end.to raise_error(GitlabCiYamlProcessor::ValidationError, "variables should be a map of key-valued strings") + end + + it "returns errors if variables is not a map of key-valued strings" do + config = YAML.dump({variables: {test: false}, rspec: {script: "test"}}) + expect do + GitlabCiYamlProcessor.new(config) + end.to raise_error(GitlabCiYamlProcessor::ValidationError, "variables should be a map of key-valued strings") + end end -end
\ No newline at end of file +end diff --git a/spec/requests/api/builds_spec.rb b/spec/requests/api/builds_spec.rb index b44798b..a169f8e 100644 --- a/spec/requests/api/builds_spec.rb +++ b/spec/requests/api/builds_spec.rb @@ -69,7 +69,10 @@ describe API::API do post api("/builds/register"), token: runner.token, info: {platform: :darwin} response.status.should == 201 - json_response["variables"].should == [{"key" => "SECRET_KEY", "value" => "secret_value"}] + json_response["variables"].should == [ + {"key" => "DB_NAME", "value" => "postgres", "public" => true}, + {"key" => "SECRET_KEY", "value" => "secret_value", "public" => false}, + ] end end diff --git a/spec/support/gitlab_stubs/gitlab_ci.yml b/spec/support/gitlab_stubs/gitlab_ci.yml index 8533cdb..3482145 100644 --- a/spec/support/gitlab_stubs/gitlab_ci.yml +++ b/spec/support/gitlab_stubs/gitlab_ci.yml @@ -7,6 +7,9 @@ before_script: - bundle install - bundle exec rake db:create +variables: + DB_NAME: postgres + types: - test - deploy |