diff options
author | Grzegorz Bizon <grzegorz@gitlab.com> | 2018-11-07 10:38:05 +0000 |
---|---|---|
committer | Grzegorz Bizon <grzegorz@gitlab.com> | 2018-11-07 10:38:05 +0000 |
commit | 6f6cd4f43dfdab19c8cb1ac7a03cae268f12b206 (patch) | |
tree | 514d15ed2eeb38a45ac7ceb820264d7e6392e58c | |
parent | b1fae097bdb54232ca56f11447ec895ea067c56c (diff) | |
parent | 7366c319df12900e40dbed679feab146d1092d89 (diff) | |
download | gitlab-ce-6f6cd4f43dfdab19c8cb1ac7a03cae268f12b206.tar.gz |
Merge branch '21480-parallel-job-keyword-mvc' into 'master'
Resolve "`parallel` job keyword MVC"
Closes #21480
See merge request gitlab-org/gitlab-ce!22631
-rw-r--r-- | app/models/ci/build.rb | 4 | ||||
-rw-r--r-- | changelogs/unreleased/21480-parallel-job-keyword-mvc.yml | 5 | ||||
-rw-r--r-- | doc/ci/variables/README.md | 2 | ||||
-rw-r--r-- | doc/ci/yaml/README.md | 23 | ||||
-rw-r--r-- | lib/gitlab/ci/config/entry/job.rb | 14 | ||||
-rw-r--r-- | lib/gitlab/ci/config/normalizer.rb | 63 | ||||
-rw-r--r-- | lib/gitlab/ci/yaml_processor.rb | 4 | ||||
-rw-r--r-- | spec/lib/gitlab/ci/config/entry/job_spec.rb | 33 | ||||
-rw-r--r-- | spec/lib/gitlab/ci/config/normalizer_spec.rb | 58 | ||||
-rw-r--r-- | spec/lib/gitlab/ci/yaml_processor_spec.rb | 27 | ||||
-rw-r--r-- | spec/models/ci/build_spec.rb | 24 |
11 files changed, 247 insertions, 10 deletions
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 360c9924a7d..25596581d0f 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -815,7 +815,7 @@ module Ci end end - def predefined_variables + def predefined_variables # rubocop:disable Metrics/AbcSize Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI', value: 'true') variables.append(key: 'GITLAB_CI', value: 'true') @@ -835,6 +835,8 @@ module Ci variables.append(key: "CI_COMMIT_TAG", value: ref) if tag? variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request variables.append(key: "CI_JOB_MANUAL", value: 'true') if action? + variables.append(key: "CI_NODE_INDEX", value: self.options[:instance].to_s) if self.options&.include?(:instance) + variables.append(key: "CI_NODE_TOTAL", value: (self.options&.dig(:parallel) || 1).to_s) variables.concat(legacy_variables) end end diff --git a/changelogs/unreleased/21480-parallel-job-keyword-mvc.yml b/changelogs/unreleased/21480-parallel-job-keyword-mvc.yml new file mode 100644 index 00000000000..7ac2410b18c --- /dev/null +++ b/changelogs/unreleased/21480-parallel-job-keyword-mvc.yml @@ -0,0 +1,5 @@ +--- +title: Implement parallel job keyword. +merge_request: 22631 +author: +type: added diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 2d23bf6d2fd..bdbcf8c9435 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -65,6 +65,8 @@ future GitLab releases.** | **CI_JOB_NAME** | 9.0 | 0.5 | The name of the job as defined in `.gitlab-ci.yml` | | **CI_JOB_STAGE** | 9.0 | 0.5 | The name of the stage as defined in `.gitlab-ci.yml` | | **CI_JOB_TOKEN** | 9.0 | 1.2 | Token used for authenticating with the [GitLab Container Registry][registry] and downloading [dependent repositories][dependent-repositories] | +| **CI_NODE_INDEX** | 11.5 | all | Index of the job in the job set. If the job is not parallelized, this variable is not set. | +| **CI_NODE_TOTAL** | 11.5 | all | Total number of instances of this job running in parallel. If the job is not parallelized, this variable is set to `1`. | | **CI_JOB_URL** | 11.1 | 0.5 | Job details URL | | **CI_REPOSITORY_URL** | 9.0 | all | The URL to clone the Git repository | | **CI_RUNNER_DESCRIPTION** | 8.10 | 0.5 | The description of the runner as saved in GitLab | diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 981aa101dd3..b3a55e48f4e 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -75,6 +75,7 @@ A job is defined by a list of parameters that define the job behavior. | environment | no | Defines a name of environment to which deployment is done by this job | | coverage | no | Define code coverage settings for a given job | | retry | no | Define how many times a job can be auto-retried in case of a failure | +| parallel | no | Defines how many instances of a job should be run in parallel | ### `extends` @@ -1451,6 +1452,26 @@ test: retry: 2 ``` +## `parallel` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22631) in GitLab 11.5. + +`parallel` allows you to configure how many instances of a job to run in +parallel. This value has to be greater than or equal to two (2). + +This creates N instances of the same job that run in parallel. They're named +sequentially from `job_name 1/N` to `job_name N/N`. + +For every job, `CI_NODE_INDEX` and `CI_NODE_TOTAL` [environment variables](../variables/README.html#predefined-variables-environment-variables) are set. + +A simple example: + +```yaml +test: + script: rspec + parallel: 5 +``` + ## `include` > Introduced in [GitLab Edition Premium][ee] 10.5. @@ -2034,4 +2055,4 @@ CI with various languages. [schedules]: ../../user/project/pipelines/schedules.md [variables-expressions]: ../variables/README.md#variables-expressions [ee]: https://about.gitlab.com/gitlab-ee/ -[gitlab-versions]: https://about.gitlab.com/products/
\ No newline at end of file +[gitlab-versions]: https://about.gitlab.com/products/ diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index e4610faa327..362014b1a09 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except type image services allow_failure type stage when start_in artifacts cache dependencies before_script after_script variables - environment coverage retry extends].freeze + environment coverage retry parallel extends].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -29,6 +29,8 @@ module Gitlab validates :retry, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 2 } + validates :parallel, numericality: { only_integer: true, + greater_than_or_equal_to: 2 } validates :when, inclusion: { in: %w[on_success on_failure always manual delayed], message: 'should be on_success, on_failure, ' \ @@ -79,17 +81,18 @@ module Gitlab description: 'Artifacts configuration for this job.' entry :environment, Entry::Environment, - description: 'Environment configuration for this job.' + description: 'Environment configuration for this job.' entry :coverage, Entry::Coverage, - description: 'Coverage configuration for this job.' + description: 'Coverage configuration for this job.' helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands, :environment, :coverage, :retry + :artifacts, :commands, :environment, :coverage, :retry, + :parallel attributes :script, :tags, :allow_failure, :when, :dependencies, - :retry, :extends, :start_in + :retry, :parallel, :extends, :start_in def compose!(deps = nil) super do @@ -158,6 +161,7 @@ module Gitlab environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, retry: retry_defined? ? retry_value.to_i : nil, + parallel: parallel_defined? ? parallel_value.to_i : nil, artifacts: artifacts_value, after_script: after_script_value, ignore: ignored? } diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb new file mode 100644 index 00000000000..969ae093e8b --- /dev/null +++ b/lib/gitlab/ci/config/normalizer.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + def initialize(jobs_config) + @jobs_config = jobs_config + end + + def normalize_jobs + extract_parallelized_jobs + parallelized_config = parallelize_jobs + parallelize_dependencies(parallelized_config) + end + + private + + def extract_parallelized_jobs + @parallelized_jobs = {} + + @jobs_config.each do |job_name, config| + if config[:parallel] + @parallelized_jobs[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) + end + end + + @parallelized_jobs + end + + def parallelize_jobs + @jobs_config.each_with_object({}) do |(job_name, config), hash| + if @parallelized_jobs.key?(job_name) + @parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) } + else + hash[job_name] = config + end + + hash + end + end + + def parallelize_dependencies(parallelized_config) + parallelized_config.each_with_object({}) do |(job_name, config), hash| + parallelized_job_names = @parallelized_jobs.keys.map(&:to_s) + if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any? + deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten + hash[job_name] = config.merge(dependencies: deps) + else + hash[job_name] = config + end + + hash + end + end + + def self.parallelize_job_names(name, total) + Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] } + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 39a1b52e531..e6ec400e476 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -52,6 +52,8 @@ module Gitlab after_script: job[:after_script], environment: job[:environment], retry: job[:retry], + parallel: job[:parallel], + instance: job[:instance], start_in: job[:start_in] }.compact } end @@ -104,7 +106,7 @@ module Gitlab ## # Jobs # - @jobs = @ci_config.jobs + @jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs @jobs.each do |name, job| # logical validation for job diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index 1169938b80c..f1a2946acda 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -1,5 +1,4 @@ -require 'fast_spec_helper' -require_dependency 'active_model' +require 'spec_helper' describe Gitlab::Ci::Config::Entry::Job do let(:entry) { described_class.new(config, name: :rspec) } @@ -138,6 +137,36 @@ describe Gitlab::Ci::Config::Entry::Job do end end + context 'when parallel value is not correct' do + context 'when it is not a numeric value' do + let(:config) { { parallel: true } } + + it 'returns error about invalid type' do + expect(entry).not_to be_valid + expect(entry.errors).to include 'job parallel is not a number' + end + end + + context 'when it is lower than two' do + let(:config) { { parallel: 1 } } + + it 'returns error about value too low' do + expect(entry).not_to be_valid + expect(entry.errors) + .to include 'job parallel must be greater than or equal to 2' + end + end + + context 'when it is not an integer' do + let(:config) { { parallel: 1.5 } } + + it 'returns error about wrong value' do + expect(entry).not_to be_valid + expect(entry.errors).to include 'job parallel must be an integer' + end + end + end + context 'when delayed job' do context 'when start_in is specified' do let(:config) { { script: 'echo', when: 'delayed', start_in: '1 day' } } diff --git a/spec/lib/gitlab/ci/config/normalizer_spec.rb b/spec/lib/gitlab/ci/config/normalizer_spec.rb new file mode 100644 index 00000000000..7c558cacdd5 --- /dev/null +++ b/spec/lib/gitlab/ci/config/normalizer_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +describe Gitlab::Ci::Config::Normalizer do + let(:job_name) { :rspec } + let(:job_config) { { script: 'rspec', parallel: 5, name: 'rspec' } } + let(:config) { { job_name => job_config } } + + describe '.normalize_jobs' do + subject { described_class.new(config).normalize_jobs } + + it 'does not have original job' do + is_expected.not_to include(job_name) + end + + it 'has parallelized jobs' do + job_names = [:"rspec 1/5", :"rspec 2/5", :"rspec 3/5", :"rspec 4/5", :"rspec 5/5"] + + is_expected.to include(*job_names) + end + + it 'sets job instance in options' do + expect(subject.values).to all(include(:instance)) + end + + it 'parallelizes jobs with original config' do + original_config = config[job_name].except(:name) + configs = subject.values.map { |config| config.except(:name, :instance) } + + expect(configs).to all(eq(original_config)) + end + + context 'when there is a job with a slash in it' do + let(:job_name) { :"rspec 35/2" } + + it 'properly parallelizes job names' do + job_names = [:"rspec 35/2 1/5", :"rspec 35/2 2/5", :"rspec 35/2 3/5", :"rspec 35/2 4/5", :"rspec 35/2 5/5"] + + is_expected.to include(*job_names) + end + end + + context 'when jobs depend on parallelized jobs' do + let(:config) { { job_name => job_config, other_job: { script: 'echo 1', dependencies: [job_name.to_s] } } } + + it 'parallelizes dependencies' do + job_names = ["rspec 1/5", "rspec 2/5", "rspec 3/5", "rspec 4/5", "rspec 5/5"] + + expect(subject[:other_job][:dependencies]).to include(*job_names) + end + + it 'does not include original job name in dependencies' do + expect(subject[:other_job][:dependencies]).not_to include(job_name) + end + end + end +end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 85b23edce9f..dcfd54107a3 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -645,6 +645,33 @@ module Gitlab end end + describe 'Parallel' do + context 'when job is parallelized' do + let(:parallel) { 5 } + + let(:config) do + YAML.dump(rspec: { script: 'rspec', + parallel: parallel }) + end + + it 'returns parallelized jobs' do + config_processor = Gitlab::Ci::YamlProcessor.new(config) + builds = config_processor.stage_builds_attributes('test') + build_options = builds.map { |build| build[:options] } + + expect(builds.size).to eq(5) + expect(build_options).to all(include(:instance, parallel: parallel)) + end + + it 'does not have the original job' do + config_processor = Gitlab::Ci::YamlProcessor.new(config) + builds = config_processor.stage_builds_attributes('test') + + expect(builds).not_to include(:rspec) + end + end + end + describe 'cache' do context 'when cache definition has unknown keys' do it 'raises relevant validation error' do diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 2e65a6a2a0f..5bd2f096656 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -2015,6 +2015,7 @@ describe Ci::Build do { key: 'CI_COMMIT_BEFORE_SHA', value: build.before_sha, public: true }, { key: 'CI_COMMIT_REF_NAME', value: build.ref, public: true }, { key: 'CI_COMMIT_REF_SLUG', value: build.ref_slug, public: true }, + { key: 'CI_NODE_TOTAL', value: '1', public: true }, { key: 'CI_BUILD_REF', value: build.sha, public: true }, { key: 'CI_BUILD_BEFORE_SHA', value: build.before_sha, public: true }, { key: 'CI_BUILD_REF_NAME', value: build.ref, public: true }, @@ -2476,6 +2477,29 @@ describe Ci::Build do end end + context 'when build is parallelized' do + let(:total) { 5 } + let(:index) { 3 } + + before do + build.options[:parallel] = total + build.options[:instance] = index + build.name = "#{build.name} #{index}/#{total}" + end + + it 'includes CI_NODE_INDEX' do + is_expected.to include( + { key: 'CI_NODE_INDEX', value: index.to_s, public: true } + ) + end + + it 'includes correct CI_NODE_TOTAL' do + is_expected.to include( + { key: 'CI_NODE_TOTAL', value: total.to_s, public: true } + ) + end + end + describe 'variables ordering' do context 'when variables hierarchy is stubbed' do let(:build_pre_var) { { key: 'build', value: 'value', public: true } } |