From 67c7e0fc5d50e2c8fc7aa773b98d32922132be47 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sun, 3 Sep 2017 23:35:37 +0900 Subject: Fail jobs if its dependency is missing --- app/models/ci/build.rb | 6 +++++ app/models/commit_status.rb | 3 ++- app/services/ci/register_job_service.rb | 3 +++ lib/gitlab/ci/error/missing_dependencies.rb | 7 +++++ .../gitlab/ci/error/missing_dependencies_spec.rb | 5 ++++ spec/models/ci/build_spec.rb | 30 ++++++++++++++++++++++ spec/services/ci/register_job_service_spec.rb | 28 ++++++++++++++++++++ 7 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/ci/error/missing_dependencies.rb create mode 100644 spec/lib/gitlab/ci/error/missing_dependencies_spec.rb diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index d2402b55184..119c6fd7b45 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -139,6 +139,12 @@ module Ci Ci::Build.retry(build, build.user) end end + + before_transition any => [:running] do |build| + if !build.empty_dependencies? && build.dependencies.empty? + raise Gitlab::Ci::Error::MissingDependencies + end + end end def detailed_status(current_user) diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index ee21ed8e420..c0263c0b4e2 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -43,7 +43,8 @@ class CommitStatus < ActiveRecord::Base script_failure: 1, api_failure: 2, stuck_or_timeout_failure: 3, - runner_system_failure: 4 + runner_system_failure: 4, + missing_dependency_failure: 5 } ## diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index 2ef76e03031..f73902935e6 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -54,6 +54,9 @@ module Ci # we still have to return 409 in the end, # to make sure that this is properly handled by runner. valid = false + rescue Gitlab::Ci::Error::MissingDependencies + build.drop!(:missing_dependency_failure) + valid = false end end diff --git a/lib/gitlab/ci/error/missing_dependencies.rb b/lib/gitlab/ci/error/missing_dependencies.rb new file mode 100644 index 00000000000..f4b1940d84f --- /dev/null +++ b/lib/gitlab/ci/error/missing_dependencies.rb @@ -0,0 +1,7 @@ +module Gitlab + module Ci + module Error + class MissingDependencies < StandardError; end + end + end +end diff --git a/spec/lib/gitlab/ci/error/missing_dependencies_spec.rb b/spec/lib/gitlab/ci/error/missing_dependencies_spec.rb new file mode 100644 index 00000000000..039a4776dc3 --- /dev/null +++ b/spec/lib/gitlab/ci/error/missing_dependencies_spec.rb @@ -0,0 +1,5 @@ +require 'spec_helper' + +describe Gitlab::Ci::Error::MissingDependencies do + it { expect(described_class).to be < StandardError } +end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 26d33663dad..61ac2dd78d1 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1868,6 +1868,36 @@ describe Ci::Build do end end + describe 'state transition: any => [:running]' do + let(:build) { create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: options) } + + context 'when "dependencies" keyword is not defined' do + let(:options) { {} } + + it { expect { build.run! }.not_to raise_error } + end + + context 'when "dependencies" keyword is empty' do + let(:options) { { dependencies: [] } } + + it { expect { build.run! }.not_to raise_error } + end + + context 'when "dependencies" keyword is specified' do + let(:options) { { dependencies: ['test'] } } + + context 'when a depended job exists' do + let!(:pre_build) { create(:ci_build, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it { expect { build.run! }.not_to raise_error } + end + + context 'when depended jobs do not exist' do + it { expect { build.run! }.to raise_error(Gitlab::Ci::Error::MissingDependencies) } + end + end + end + describe 'state transition when build fails' do let(:service) { MergeRequests::AddTodoWhenBuildFailsService.new(project, user) } diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb index decdd577226..e779d02cc52 100644 --- a/spec/services/ci/register_job_service_spec.rb +++ b/spec/services/ci/register_job_service_spec.rb @@ -276,6 +276,34 @@ module Ci end end + context 'when "dependencies" keyword is specified' do + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: job_name, stage_idx: 0) } + + let!(:pending_job) do + create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: { dependencies: ['spec'] } ) + end + + let(:picked_job) { execute(specific_runner) } + + context 'when a depended job exists' do + let(:job_name) { 'spec' } + + it "picks the build" do + expect(picked_job).to eq(pending_job) + end + end + + context 'when depended jobs do not exist' do + let(:job_name) { 'robocop' } + + it 'does not pick the build and drops the build' do + expect(picked_job).to be_nil + expect(pending_job.reload).to be_failed + expect(pending_job).to be_missing_dependency_failure + end + end + end + def execute(runner) described_class.new(runner).execute.build end -- cgit v1.2.1 From 8917726bb5e6746750129c0d9322c2daa7f88172 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sun, 3 Sep 2017 23:45:44 +0900 Subject: Add changelog. Fix doc --- .../feature-sm-34834-missing-dependency-should-fail-job-2.yml | 5 +++++ doc/ci/yaml/README.md | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 changelogs/unreleased/feature-sm-34834-missing-dependency-should-fail-job-2.yml diff --git a/changelogs/unreleased/feature-sm-34834-missing-dependency-should-fail-job-2.yml b/changelogs/unreleased/feature-sm-34834-missing-dependency-should-fail-job-2.yml new file mode 100644 index 00000000000..ab85b8ee515 --- /dev/null +++ b/changelogs/unreleased/feature-sm-34834-missing-dependency-should-fail-job-2.yml @@ -0,0 +1,5 @@ +--- +title: Fail jobs if its dependency is missing +merge_request: 14009 +author: +type: fixed diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index f40d2c5e347..ef32e7658ee 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -1106,7 +1106,8 @@ Note that `artifacts` from all previous [stages](#stages) are passed by default. To use this feature, define `dependencies` in context of the job and pass a list of all previous jobs from which the artifacts should be downloaded. You can only define jobs from stages that are executed before the current one. -An error will be shown if you define jobs from the current stage or next ones. +An error will be shown if you define jobs from the current stage or next ones, +or there are no depended jobs in previous stages. Defining an empty array will skip downloading any artifacts for that job. The status of the previous job is not considered when using `dependencies`, so if it failed or it is a manual job that was not run, no error occurs. -- cgit v1.2.1 From 6e343b27bfb993b2c19dd4b4fd8d2b48747fbac3 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 4 Sep 2017 17:56:49 +0900 Subject: Use Class.new(StandardError) instead of custom extended error class. Bring back specified_dependencies?. --- app/models/ci/build.rb | 10 ++++++++-- app/services/ci/register_job_service.rb | 2 +- lib/gitlab/ci/error/missing_dependencies.rb | 7 ------- spec/lib/gitlab/ci/error/missing_dependencies_spec.rb | 5 ----- spec/models/ci/build_spec.rb | 2 +- 5 files changed, 10 insertions(+), 16 deletions(-) delete mode 100644 lib/gitlab/ci/error/missing_dependencies.rb delete mode 100644 spec/lib/gitlab/ci/error/missing_dependencies_spec.rb diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 119c6fd7b45..965ba35c8b0 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -6,6 +6,8 @@ module Ci include Presentable include Importable + MissingDependenciesError = Class.new(StandardError) + belongs_to :runner belongs_to :trigger_request belongs_to :erased_by, class_name: 'User' @@ -141,8 +143,8 @@ module Ci end before_transition any => [:running] do |build| - if !build.empty_dependencies? && build.dependencies.empty? - raise Gitlab::Ci::Error::MissingDependencies + if build.specified_dependencies? && build.dependencies.empty? + raise MissingDependenciesError end end end @@ -484,6 +486,10 @@ module Ci options[:dependencies]&.empty? end + def specified_dependencies? + options.has_key?(:dependencies) && options[:dependencies].any? + end + def hide_secrets(trace) return unless trace diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index f73902935e6..c8b6450c9b5 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -54,7 +54,7 @@ module Ci # we still have to return 409 in the end, # to make sure that this is properly handled by runner. valid = false - rescue Gitlab::Ci::Error::MissingDependencies + rescue Ci::Build::MissingDependenciesError build.drop!(:missing_dependency_failure) valid = false end diff --git a/lib/gitlab/ci/error/missing_dependencies.rb b/lib/gitlab/ci/error/missing_dependencies.rb deleted file mode 100644 index f4b1940d84f..00000000000 --- a/lib/gitlab/ci/error/missing_dependencies.rb +++ /dev/null @@ -1,7 +0,0 @@ -module Gitlab - module Ci - module Error - class MissingDependencies < StandardError; end - end - end -end diff --git a/spec/lib/gitlab/ci/error/missing_dependencies_spec.rb b/spec/lib/gitlab/ci/error/missing_dependencies_spec.rb deleted file mode 100644 index 039a4776dc3..00000000000 --- a/spec/lib/gitlab/ci/error/missing_dependencies_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'spec_helper' - -describe Gitlab::Ci::Error::MissingDependencies do - it { expect(described_class).to be < StandardError } -end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 61ac2dd78d1..f8d8b1800b8 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1893,7 +1893,7 @@ describe Ci::Build do end context 'when depended jobs do not exist' do - it { expect { build.run! }.to raise_error(Gitlab::Ci::Error::MissingDependencies) } + it { expect { build.run! }.to raise_error(Ci::Build::MissingDependenciesError) } end end end -- cgit v1.2.1 From c3e0731d2efc777018b668d9e0b7f8aa2377d9fc Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 5 Sep 2017 22:37:28 +0900 Subject: Add case when artifacts have not existed on dependencies --- app/models/ci/build.rb | 20 +++++++++++++++----- doc/ci/yaml/README.md | 2 +- spec/models/ci/build_spec.rb | 21 ++++++++++++++++++++- spec/services/ci/register_job_service_spec.rb | 25 +++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 965ba35c8b0..9c44e9ef5fd 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -143,9 +143,7 @@ module Ci end before_transition any => [:running] do |build| - if build.specified_dependencies? && build.dependencies.empty? - raise MissingDependenciesError - end + build.validates_dependencies! end end @@ -486,8 +484,20 @@ module Ci options[:dependencies]&.empty? end - def specified_dependencies? - options.has_key?(:dependencies) && options[:dependencies].any? + def validates_dependencies! + dependencies.tap do |deps| + # When `dependencies` keyword is given and depended jobs are skipped by `only` keyword + if options[:dependencies]&.any? && deps.empty? + raise MissingDependenciesError + end + + # When artifacts of depended jobs have not existsed + deps.each do |dep| + if dep.options[:artifacts]&.any? && !dep.artifacts? + raise MissingDependenciesError + end + end + end end def hide_secrets(trace) diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index ef32e7658ee..f5391ff0768 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -1107,7 +1107,7 @@ To use this feature, define `dependencies` in context of the job and pass a list of all previous jobs from which the artifacts should be downloaded. You can only define jobs from stages that are executed before the current one. An error will be shown if you define jobs from the current stage or next ones, -or there are no depended jobs in previous stages. +or there are no depended jobs with artifacts in previous stages. Defining an empty array will skip downloading any artifacts for that job. The status of the previous job is not considered when using `dependencies`, so if it failed or it is a manual job that was not run, no error occurs. diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index f8d8b1800b8..230546cf2da 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1887,9 +1887,28 @@ describe Ci::Build do let(:options) { { dependencies: ['test'] } } context 'when a depended job exists' do - let!(:pre_build) { create(:ci_build, pipeline: pipeline, name: 'test', stage_idx: 0) } + let!(:pre_stage_job) { create(:ci_build, pipeline: pipeline, name: 'test', stage_idx: 0) } it { expect { build.run! }.not_to raise_error } + + context 'when "artifacts" keyword is specified on depended job' do + let!(:pre_stage_job) do + create(:ci_build, :artifacts, pipeline: pipeline, name: 'test', stage_idx: 0, + options: { artifacts: { paths: ['binaries/'] } } ) + end + + context 'when artifacts of depended job has existsed' do + it { expect { build.run! }.not_to raise_error } + end + + context 'when artifacts of depended job has not existsed' do + before do + pre_stage_job.erase_artifacts! + end + + it { expect { build.run! }.to raise_error(Ci::Build::MissingDependenciesError) } + end + end end context 'when depended jobs do not exist' do diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb index e779d02cc52..b5f88d6cdbe 100644 --- a/spec/services/ci/register_job_service_spec.rb +++ b/spec/services/ci/register_job_service_spec.rb @@ -291,6 +291,31 @@ module Ci it "picks the build" do expect(picked_job).to eq(pending_job) end + + context 'when "artifacts" keyword is specified on depended job' do + let!(:pre_stage_job) do + create(:ci_build, :success, :artifacts, pipeline: pipeline, name: job_name, stage_idx: 0, + options: { artifacts: { paths: ['binaries/'] } } ) + end + + context 'when artifacts of depended job has existsed' do + it "picks the build" do + expect(picked_job).to eq(pending_job) + end + end + + context 'when artifacts of depended job has not existsed' do + before do + pre_stage_job.erase_artifacts! + end + + it 'does not pick the build and drops the build' do + expect(picked_job).to be_nil + expect(pending_job.reload).to be_failed + expect(pending_job).to be_missing_dependency_failure + end + end + end end context 'when depended jobs do not exist' do -- cgit v1.2.1 From f5bfedc6122e00ed1182ad7caaeb7636eeebebe1 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 6 Sep 2017 00:51:05 +0900 Subject: Fix lint --- spec/models/ci/build_spec.rb | 8 ++++++-- spec/services/ci/register_job_service_spec.rb | 9 +++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 230546cf2da..6d33d0d917a 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1893,8 +1893,12 @@ describe Ci::Build do context 'when "artifacts" keyword is specified on depended job' do let!(:pre_stage_job) do - create(:ci_build, :artifacts, pipeline: pipeline, name: 'test', stage_idx: 0, - options: { artifacts: { paths: ['binaries/'] } } ) + create(:ci_build, + :artifacts, + pipeline: pipeline, + name: 'test', + stage_idx: 0, + options: { artifacts: { paths: ['binaries/'] } } ) end context 'when artifacts of depended job has existsed' do diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb index b5f88d6cdbe..b86b9d7a42b 100644 --- a/spec/services/ci/register_job_service_spec.rb +++ b/spec/services/ci/register_job_service_spec.rb @@ -294,8 +294,13 @@ module Ci context 'when "artifacts" keyword is specified on depended job' do let!(:pre_stage_job) do - create(:ci_build, :success, :artifacts, pipeline: pipeline, name: job_name, stage_idx: 0, - options: { artifacts: { paths: ['binaries/'] } } ) + create(:ci_build, + :success, + :artifacts, + pipeline: pipeline, + name: job_name, + stage_idx: 0, + options: { artifacts: { paths: ['binaries/'] } } ) end context 'when artifacts of depended job has existsed' do -- cgit v1.2.1 From fba38b51b761cdc5edb964dc90eea051a33aab3e Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 27 Nov 2017 18:59:03 +0900 Subject: Add feature flag --- app/models/ci/build.rb | 2 ++ doc/ci/yaml/README.md | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 9c44e9ef5fd..cf666f86841 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -485,6 +485,8 @@ module Ci end def validates_dependencies! + return unless Feature.enabled?('ci_validates_dependencies') + dependencies.tap do |deps| # When `dependencies` keyword is given and depended jobs are skipped by `only` keyword if options[:dependencies]&.any? && deps.empty? diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index f5391ff0768..ea151853f50 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -1106,8 +1106,7 @@ Note that `artifacts` from all previous [stages](#stages) are passed by default. To use this feature, define `dependencies` in context of the job and pass a list of all previous jobs from which the artifacts should be downloaded. You can only define jobs from stages that are executed before the current one. -An error will be shown if you define jobs from the current stage or next ones, -or there are no depended jobs with artifacts in previous stages. +An error will be shown if you define jobs from the current stage or next ones. Defining an empty array will skip downloading any artifacts for that job. The status of the previous job is not considered when using `dependencies`, so if it failed or it is a manual job that was not run, no error occurs. @@ -1154,6 +1153,22 @@ deploy: script: make deploy ``` +### Validations for `dependencies` keyword + +> Introduced in GitLab 10.3 + +`dependencies` keyword doesn't check the depended `artifacts` strictly. Therefore +they do not fail even though it falls into the following conditions. + +1. A depended `artifacts` has been [erased](https://docs.gitlab.com/ee/api/jobs.html#erase-a-job). +1. A depended `artifacts` has been [expired](https://docs.gitlab.com/ee/ci/yaml/#artifacts-expire_in). + +To validate those conditions, you can flip the feature flag from a rails console: + +``` +Feature.enable('ci_validates_dependencies') +``` + ### before_script and after_script It's possible to overwrite the globally defined `before_script` and `after_script`: -- cgit v1.2.1 From 38d46754be49f13c1f92fd1f79ff49c76ec55c49 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 2 Dec 2017 16:14:47 +0900 Subject: Optimize valid_dependency method by ayufan thought --- app/models/ci/build.rb | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index cf666f86841..a29fb0ad2ca 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -143,7 +143,7 @@ module Ci end before_transition any => [:running] do |build| - build.validates_dependencies! + build.validates_dependencies! if Feature.enabled?('ci_validates_dependencies') end end @@ -485,20 +485,8 @@ module Ci end def validates_dependencies! - return unless Feature.enabled?('ci_validates_dependencies') - - dependencies.tap do |deps| - # When `dependencies` keyword is given and depended jobs are skipped by `only` keyword - if options[:dependencies]&.any? && deps.empty? - raise MissingDependenciesError - end - - # When artifacts of depended jobs have not existsed - deps.each do |dep| - if dep.options[:artifacts]&.any? && !dep.artifacts? - raise MissingDependenciesError - end - end + dependencies.each do |dependency| + raise MissingDependenciesError unless dependency.valid_dependency? end end @@ -612,5 +600,13 @@ module Ci update_project_statistics end end + + def valid_dependency? + return false unless complete? + return false if artifacts_expired? + return false if erased? + + true + end end end -- cgit v1.2.1 From 6171db2d2df337ef52460387a48f28136e809861 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 2 Dec 2017 16:23:19 +0900 Subject: Fix /build_spec.rb --- app/models/ci/build.rb | 16 ++++++++-------- spec/models/ci/build_spec.rb | 7 ++++++- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index a29fb0ad2ca..fbda0962a91 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -490,6 +490,14 @@ module Ci end end + def valid_dependency? + return false unless complete? + return false if artifacts_expired? + return false if erased? + + true + end + def hide_secrets(trace) return unless trace @@ -600,13 +608,5 @@ module Ci update_project_statistics end end - - def valid_dependency? - return false unless complete? - return false if artifacts_expired? - return false if erased? - - true - end end end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 6d33d0d917a..2cacf04a791 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1869,6 +1869,10 @@ describe Ci::Build do end describe 'state transition: any => [:running]' do + before do + stub_feature_flags(ci_validates_dependencies: true) + end + let(:build) { create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: options) } context 'when "dependencies" keyword is not defined' do @@ -1887,13 +1891,14 @@ describe Ci::Build do let(:options) { { dependencies: ['test'] } } context 'when a depended job exists' do - let!(:pre_stage_job) { create(:ci_build, pipeline: pipeline, name: 'test', stage_idx: 0) } + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0) } it { expect { build.run! }.not_to raise_error } context 'when "artifacts" keyword is specified on depended job' do let!(:pre_stage_job) do create(:ci_build, + :success, :artifacts, pipeline: pipeline, name: 'test', -- cgit v1.2.1 From cd23850e0002f6c39b0c7f97c1dc484d7993af04 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 6 Dec 2017 16:40:56 +0900 Subject: Fix tests --- spec/models/ci/build_spec.rb | 44 ++++++++++++++++----------- spec/services/ci/register_job_service_spec.rb | 16 +++------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 2cacf04a791..96d9fba2a2d 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1891,11 +1891,7 @@ describe Ci::Build do let(:options) { { dependencies: ['test'] } } context 'when a depended job exists' do - let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0) } - - it { expect { build.run! }.not_to raise_error } - - context 'when "artifacts" keyword is specified on depended job' do + context 'when depended job has artifacts' do let!(:pre_stage_job) do create(:ci_build, :success, @@ -1906,22 +1902,36 @@ describe Ci::Build do options: { artifacts: { paths: ['binaries/'] } } ) end - context 'when artifacts of depended job has existsed' do - it { expect { build.run! }.not_to raise_error } - end + it { expect { build.run! }.not_to raise_error } + end - context 'when artifacts of depended job has not existsed' do - before do - pre_stage_job.erase_artifacts! - end + context 'when depended job does not have artifacts' do + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0) } - it { expect { build.run! }.to raise_error(Ci::Build::MissingDependenciesError) } - end + it { expect { build.run! }.not_to raise_error } end - end - context 'when depended jobs do not exist' do - it { expect { build.run! }.to raise_error(Ci::Build::MissingDependenciesError) } + context 'when depended job has not been completed yet' do + let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it { expect { build.run! }.to raise_error(Ci::Build::MissingDependenciesError) } + end + + context 'when artifacts of depended job has been expired' do + let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it { expect { build.run! }.to raise_error(Ci::Build::MissingDependenciesError) } + end + + context 'when artifacts of depended job has been erased' do + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) } + + before do + pre_stage_job.erase + end + + it { expect { build.run! }.to raise_error(Ci::Build::MissingDependenciesError) } + end end end end diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb index b86b9d7a42b..19e78faa591 100644 --- a/spec/services/ci/register_job_service_spec.rb +++ b/spec/services/ci/register_job_service_spec.rb @@ -277,6 +277,10 @@ module Ci end context 'when "dependencies" keyword is specified' do + before do + stub_feature_flags(ci_validates_dependencies: true) + end + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: job_name, stage_idx: 0) } let!(:pending_job) do @@ -311,7 +315,7 @@ module Ci context 'when artifacts of depended job has not existsed' do before do - pre_stage_job.erase_artifacts! + pre_stage_job.erase end it 'does not pick the build and drops the build' do @@ -322,16 +326,6 @@ module Ci end end end - - context 'when depended jobs do not exist' do - let(:job_name) { 'robocop' } - - it 'does not pick the build and drops the build' do - expect(picked_job).to be_nil - expect(pending_job.reload).to be_failed - expect(pending_job).to be_missing_dependency_failure - end - end end def execute(runner) -- cgit v1.2.1 From 3b11df7bee6d574488a394b46ced22a824d5081d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 6 Dec 2017 21:24:11 +0900 Subject: Fix pipeline --- spec/models/ci/pipeline_spec.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index d4b1e7c8dd4..bb89e093890 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -1244,7 +1244,7 @@ describe Ci::Pipeline, :mailer do describe '#execute_hooks' do let!(:build_a) { create_build('a', 0) } - let!(:build_b) { create_build('b', 1) } + let!(:build_b) { create_build('b', 0) } let!(:hook) do create(:project_hook, project: project, pipeline_events: enabled) @@ -1300,6 +1300,8 @@ describe Ci::Pipeline, :mailer do end context 'when stage one failed' do + let!(:build_b) { create_build('b', 1) } + before do build_a.drop end -- cgit v1.2.1 From c2f68b7ae319cb732eeaf53ddd020702644dc70d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 7 Dec 2017 23:16:57 +0900 Subject: Update document to use ci_validates_dependencies --- doc/ci/yaml/README.md | 14 ++++---------- doc/user/project/pipelines/job_artifacts.md | 8 ++++++++ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index ea151853f50..ac5d99c71fc 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -1153,22 +1153,16 @@ deploy: script: make deploy ``` -### Validations for `dependencies` keyword - +>**Note:** > Introduced in GitLab 10.3 +> This is the user documentation. For the administration guide see + [administration/job_artifacts](../../../administration/job_artifacts.md#validation_for_dependency). -`dependencies` keyword doesn't check the depended `artifacts` strictly. Therefore -they do not fail even though it falls into the following conditions. +If a depended job doesn't have artifacts by the following reason, the depending job will fail. 1. A depended `artifacts` has been [erased](https://docs.gitlab.com/ee/api/jobs.html#erase-a-job). 1. A depended `artifacts` has been [expired](https://docs.gitlab.com/ee/ci/yaml/#artifacts-expire_in). -To validate those conditions, you can flip the feature flag from a rails console: - -``` -Feature.enable('ci_validates_dependencies') -``` - ### before_script and after_script It's possible to overwrite the globally defined `before_script` and `after_script`: diff --git a/doc/user/project/pipelines/job_artifacts.md b/doc/user/project/pipelines/job_artifacts.md index f9a268fb789..f8675f77856 100644 --- a/doc/user/project/pipelines/job_artifacts.md +++ b/doc/user/project/pipelines/job_artifacts.md @@ -163,6 +163,14 @@ information in the UI. ![Latest artifacts button](img/job_latest_artifacts_browser.png) +## Validation for `dependency` keyword + +To disable [the validation for dependency], you can flip the feature flag from a rails console: + +``` +Feature.enable('ci_disable_validates_dependencies') +``` [expiry date]: ../../../ci/yaml/README.md#artifacts-expire_in +[the validation for dependency]: ../../../ci/yaml/README.md#dependencies [ce-14399]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14399 -- cgit v1.2.1 From 85151ff6f65fcb3076b554c563ccc2fa458a9ccb Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 7 Dec 2017 23:17:46 +0900 Subject: Change feature flag to ci_disable_validates_dependencies to enable it as default --- app/models/ci/build.rb | 2 +- spec/models/ci/build_spec.rb | 2 +- spec/services/ci/register_job_service_spec.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index fbda0962a91..85960f1b6bb 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -143,7 +143,7 @@ module Ci end before_transition any => [:running] do |build| - build.validates_dependencies! if Feature.enabled?('ci_validates_dependencies') + build.validates_dependencies! unless Feature.enabled?('ci_disable_validates_dependencies') end end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 96d9fba2a2d..3440ce7f1e8 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1870,7 +1870,7 @@ describe Ci::Build do describe 'state transition: any => [:running]' do before do - stub_feature_flags(ci_validates_dependencies: true) + stub_feature_flags(ci_disable_validates_dependencies: true) end let(:build) { create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: options) } diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb index 19e78faa591..16218b78fb8 100644 --- a/spec/services/ci/register_job_service_spec.rb +++ b/spec/services/ci/register_job_service_spec.rb @@ -278,7 +278,7 @@ module Ci context 'when "dependencies" keyword is specified' do before do - stub_feature_flags(ci_validates_dependencies: true) + stub_feature_flags(ci_disable_validates_dependencies: false) end let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: job_name, stage_idx: 0) } -- cgit v1.2.1 From 30bc983c340de3641b952606c154c7a413368ab6 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 8 Dec 2017 02:21:16 +0900 Subject: Test for both ci_disable_validates_dependencies true/false --- spec/models/ci/build_spec.rb | 108 +++++++++++++++----------- spec/services/ci/register_job_service_spec.rb | 99 +++++++++++++++-------- 2 files changed, 129 insertions(+), 78 deletions(-) diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 3440ce7f1e8..a6258676767 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1869,71 +1869,91 @@ describe Ci::Build do end describe 'state transition: any => [:running]' do - before do - stub_feature_flags(ci_disable_validates_dependencies: true) - end + shared_examples 'validation is active' do + context 'when depended job has not been completed yet' do + let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) } - let(:build) { create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: options) } + it { expect { job.run! }.to raise_error(Ci::Build::MissingDependenciesError) } + end - context 'when "dependencies" keyword is not defined' do - let(:options) { {} } + context 'when artifacts of depended job has been expired' do + let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) } - it { expect { build.run! }.not_to raise_error } - end + it { expect { job.run! }.to raise_error(Ci::Build::MissingDependenciesError) } + end - context 'when "dependencies" keyword is empty' do - let(:options) { { dependencies: [] } } + context 'when artifacts of depended job has been erased' do + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) } - it { expect { build.run! }.not_to raise_error } + before do + pre_stage_job.erase + end + + it { expect { job.run! }.to raise_error(Ci::Build::MissingDependenciesError) } + end end - context 'when "dependencies" keyword is specified' do - let(:options) { { dependencies: ['test'] } } + shared_examples 'validation is not active' do + context 'when depended job has not been completed yet' do + let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) } - context 'when a depended job exists' do - context 'when depended job has artifacts' do - let!(:pre_stage_job) do - create(:ci_build, - :success, - :artifacts, - pipeline: pipeline, - name: 'test', - stage_idx: 0, - options: { artifacts: { paths: ['binaries/'] } } ) - end + it { expect { job.run! }.not_to raise_error } + end - it { expect { build.run! }.not_to raise_error } - end + context 'when artifacts of depended job has been expired' do + let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it { expect { job.run! }.not_to raise_error } + end - context 'when depended job does not have artifacts' do - let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0) } + context 'when artifacts of depended job has been erased' do + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) } - it { expect { build.run! }.not_to raise_error } + before do + pre_stage_job.erase end - context 'when depended job has not been completed yet' do - let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) } + it { expect { job.run! }.not_to raise_error } + end + end - it { expect { build.run! }.to raise_error(Ci::Build::MissingDependenciesError) } - end + let!(:job) { create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: options) } - context 'when artifacts of depended job has been expired' do - let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) } + context 'when validates for dependencies is enabled' do + before do + stub_feature_flags(ci_disable_validates_dependencies: false) + end - it { expect { build.run! }.to raise_error(Ci::Build::MissingDependenciesError) } - end + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0) } - context 'when artifacts of depended job has been erased' do - let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) } + context 'when "dependencies" keyword is not defined' do + let(:options) { {} } - before do - pre_stage_job.erase - end + it { expect { job.run! }.not_to raise_error } + end - it { expect { build.run! }.to raise_error(Ci::Build::MissingDependenciesError) } - end + context 'when "dependencies" keyword is empty' do + let(:options) { { dependencies: [] } } + + it { expect { job.run! }.not_to raise_error } + end + + context 'when "dependencies" keyword is specified' do + let(:options) { { dependencies: ['test'] } } + + it_behaves_like 'validation is active' end end + + context 'when validates for dependencies is disabled' do + let(:options) { { dependencies: ['test'] } } + + before do + stub_feature_flags(ci_disable_validates_dependencies: true) + end + + it_behaves_like 'validation is not active' + end end describe 'state transition when build fails' do diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb index 16218b78fb8..3ee59014b5b 100644 --- a/spec/services/ci/register_job_service_spec.rb +++ b/spec/services/ci/register_job_service_spec.rb @@ -277,54 +277,85 @@ module Ci end context 'when "dependencies" keyword is specified' do - before do - stub_feature_flags(ci_disable_validates_dependencies: false) + shared_examples 'not pick' do + it 'does not pick the build and drops the build' do + expect(subject).to be_nil + expect(pending_job.reload).to be_failed + expect(pending_job).to be_missing_dependency_failure + end end - let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: job_name, stage_idx: 0) } + shared_examples 'validation is active' do + context 'when depended job has not been completed yet' do + let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it_behaves_like 'not pick' + end + + context 'when artifacts of depended job has been expired' do + let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it_behaves_like 'not pick' + end + + context 'when artifacts of depended job has been erased' do + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) } + + before do + pre_stage_job.erase + end - let!(:pending_job) do - create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: { dependencies: ['spec'] } ) + it_behaves_like 'not pick' + end end - let(:picked_job) { execute(specific_runner) } + shared_examples 'validation is not active' do + context 'when depended job has not been completed yet' do + let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) } - context 'when a depended job exists' do - let(:job_name) { 'spec' } + it { expect(subject).to eq(pending_job) } + end + + context 'when artifacts of depended job has been expired' do + let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) } - it "picks the build" do - expect(picked_job).to eq(pending_job) + it { expect(subject).to eq(pending_job) } end - context 'when "artifacts" keyword is specified on depended job' do - let!(:pre_stage_job) do - create(:ci_build, - :success, - :artifacts, - pipeline: pipeline, - name: job_name, - stage_idx: 0, - options: { artifacts: { paths: ['binaries/'] } } ) - end + context 'when artifacts of depended job has been erased' do + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) } - context 'when artifacts of depended job has existsed' do - it "picks the build" do - expect(picked_job).to eq(pending_job) - end + before do + pre_stage_job.erase end - context 'when artifacts of depended job has not existsed' do - before do - pre_stage_job.erase - end + it { expect(subject).to eq(pending_job) } + end + end - it 'does not pick the build and drops the build' do - expect(picked_job).to be_nil - expect(pending_job.reload).to be_failed - expect(pending_job).to be_missing_dependency_failure - end - end + before do + stub_feature_flags(ci_disable_validates_dependencies: false) + end + + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0) } + let!(:pending_job) { create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: { dependencies: ['test'] } ) } + + subject { execute(specific_runner) } + + context 'when validates for dependencies is enabled' do + before do + stub_feature_flags(ci_disable_validates_dependencies: false) end + + it_behaves_like 'validation is active' + end + + context 'when validates for dependencies is disabled' do + before do + stub_feature_flags(ci_disable_validates_dependencies: true) + end + + it_behaves_like 'validation is not active' end end -- cgit v1.2.1 From 2ac6d806900f3aea708b3fcdc32463235f83eb73 Mon Sep 17 00:00:00 2001 From: Achilleas Pipinellis Date: Thu, 7 Dec 2017 19:46:27 +0100 Subject: Clean up docs for dependencies validation --- doc/administration/job_artifacts.md | 39 +++++++++++++++++++++++++++++ doc/ci/yaml/README.md | 18 +++++++------ doc/user/project/pipelines/job_artifacts.md | 22 +++++++++------- 3 files changed, 63 insertions(+), 16 deletions(-) diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index 86b436d89dd..33f8a69c249 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -128,6 +128,45 @@ steps below. 1. Save the file and [restart GitLab][] for the changes to take effect. +## Validation for dependencies + +> Introduced in GitLab 10.3. + +To disable [the dependencies validation](../ci/yaml/README.md#when-a-dependent-job-will-fail), +you can flip the feature flag from a Rails console. + +--- + +**In Omnibus installations:** + +1. Enter the Rails console: + + ```sh + sudo gitlab-rails console + ``` + +1. Flip the switch and disable it: + + ```ruby + Feature.enable('ci_disable_validates_dependencies') + ``` +--- + +**In installations from source:** + +1. Enter the Rails console: + + ```sh + cd /home/git/gitlab + RAILS_ENV=production sudo -u git -H bundle exec rails console + ``` + +1. Flip the switch and disable it: + + ```ruby + Feature.enable('ci_disable_validates_dependencies') + ``` + ## Set the maximum file size of the artifacts Provided the artifacts are enabled, you can change the maximum file size of the diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index ac5d99c71fc..32464cbb259 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -1153,15 +1153,19 @@ deploy: script: make deploy ``` ->**Note:** -> Introduced in GitLab 10.3 -> This is the user documentation. For the administration guide see - [administration/job_artifacts](../../../administration/job_artifacts.md#validation_for_dependency). +#### When a dependent job will fail + +> Introduced in GitLab 10.3. -If a depended job doesn't have artifacts by the following reason, the depending job will fail. +If the artifacts of the job that is set as a dependency have been +[expired](#artifacts-expire_in) or +[erased](../../user/project/pipelines/job_artifacts.md#erasing-artifacts), then +the dependent job will fail. -1. A depended `artifacts` has been [erased](https://docs.gitlab.com/ee/api/jobs.html#erase-a-job). -1. A depended `artifacts` has been [expired](https://docs.gitlab.com/ee/ci/yaml/#artifacts-expire_in). +NOTE: **Note:** +You can ask your administrator to +[flip this switch](../../administration/job_artifacts.md#validation-for-dependencies) +and bring back the old behavior. ### before_script and after_script diff --git a/doc/user/project/pipelines/job_artifacts.md b/doc/user/project/pipelines/job_artifacts.md index f8675f77856..402989f4508 100644 --- a/doc/user/project/pipelines/job_artifacts.md +++ b/doc/user/project/pipelines/job_artifacts.md @@ -44,7 +44,7 @@ the artifacts will be kept forever. For more examples on artifacts, follow the [artifacts reference in `.gitlab-ci.yml`](../../../ci/yaml/README.md#artifacts). -## Browsing job artifacts +## Browsing artifacts >**Note:** With GitLab 9.2, PDFs, images, videos and other formats can be previewed @@ -77,7 +77,7 @@ one HTML file that you can view directly online when --- -## Downloading job artifacts +## Downloading artifacts If you need to download the whole archive, there are buttons in various places inside GitLab that make that possible. @@ -102,7 +102,7 @@ inside GitLab that make that possible. ![Job artifacts browser](img/job_artifacts_browser.png) -## Downloading the latest job artifacts +## Downloading the latest artifacts It is possible to download the latest artifacts of a job via a well known URL so you can use it for scripting purposes. @@ -163,14 +163,18 @@ information in the UI. ![Latest artifacts button](img/job_latest_artifacts_browser.png) -## Validation for `dependency` keyword +## Erasing artifacts -To disable [the validation for dependency], you can flip the feature flag from a rails console: +DANGER: **Warning:** +This is a destructive action that leads to data loss. Use with caution. -``` -Feature.enable('ci_disable_validates_dependencies') -``` +If you have at least Developer [permissions](../../permissions.md#gitlab-ci-cd-permissions) +on the project, you can erase a single job via the UI which will also remove the +artifacts and the job's trace. + +1. Navigate to a job's page. +1. Click the trash icon at the top right of the job's trace. +1. Confirm the deletion. [expiry date]: ../../../ci/yaml/README.md#artifacts-expire_in -[the validation for dependency]: ../../../ci/yaml/README.md#dependencies [ce-14399]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14399 -- cgit v1.2.1