diff options
22 files changed, 221 insertions, 65 deletions
diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 29aa00a66d9..5450d40ea95 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -2,11 +2,13 @@ module Ci class Bridge < CommitStatus + include Ci::Processable include Importable include AfterCommitQueue include Gitlab::Utils::StrongMemoize belongs_to :project + belongs_to :trigger_request validates :ref, presence: true def self.retry(bridge, current_user) @@ -23,6 +25,21 @@ module Ci .fabricate! end + def schedulable? + false + end + + def action? + false + end + + def artifacts? + false + end + + def expanded_environment_name + end + def predefined_variables raise NotImplementedError end @@ -30,5 +47,9 @@ module Ci def execute_hooks raise NotImplementedError end + + def to_partial_path + 'projects/generic_commit_statuses/generic_commit_status' + end end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index cfdb3c0d719..35cf4f8d277 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -3,6 +3,7 @@ module Ci class Build < CommitStatus prepend ArtifactMigratable + include Ci::Processable include TokenAuthenticatable include AfterCommitQueue include ObjectStorage::BackgroundMove @@ -638,10 +639,6 @@ module Ci super || project.try(:build_coverage_regex) end - def when - read_attribute(:when) || 'on_success' - end - def options read_metadata_attribute(:options, :config_options, {}) end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 30a957b4117..acef5d2e643 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -25,6 +25,8 @@ module Ci has_many :stages, -> { order(position: :asc) }, inverse_of: :pipeline has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline + has_many :processables, -> { processables }, + class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent has_many :variables, class_name: 'Ci::PipelineVariable' diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 58f3fe2460a..0389945191e 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -14,6 +14,7 @@ module Ci has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id has_many :builds, foreign_key: :stage_id + has_many :bridges, foreign_key: :stage_id with_options unless: :importing? do validates :project, presence: true diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 0f50bd39131..7f6562b63e5 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -41,6 +41,7 @@ class CommitStatus < ActiveRecord::Base scope :latest_ordered, -> { latest.ordered.includes(project: :namespace) } scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) } scope :after_stage, -> (index) { where('stage_idx > ?', index) } + scope :processables, -> { where(type: %w[Ci::Build Ci::Bridge]) } # We use `CommitStatusEnums.failure_reasons` here so that EE can more easily # extend this `Hash` with new values. diff --git a/app/models/concerns/ci/processable.rb b/app/models/concerns/ci/processable.rb new file mode 100644 index 00000000000..1c78b1413a8 --- /dev/null +++ b/app/models/concerns/ci/processable.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Ci + ## + # This module implements methods that need to be implemented by CI/CD + # entities that are supposed to go through pipeline processing + # services. + # + # + module Processable + def schedulable? + raise NotImplementedError + end + + def action? + raise NotImplementedError + end + + def when + read_attribute(:when) || 'on_success' + end + + def expanded_environment_name + raise NotImplementedError + end + end +end diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 446188347df..4a7ce00b8e2 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -10,7 +10,7 @@ module Ci update_retried new_builds = - stage_indexes_of_created_builds.map do |index| + stage_indexes_of_created_processables.map do |index| process_stage(index) end @@ -27,7 +27,7 @@ module Ci return if HasStatus::BLOCKED_STATUS.include?(current_status) if HasStatus::COMPLETED_STATUSES.include?(current_status) - created_builds_in_stage(index).select do |build| + created_processables_in_stage(index).select do |build| Gitlab::OptimisticLocking.retry_lock(build) do |subject| Ci::ProcessBuildService.new(project, @user) .execute(build, current_status) @@ -43,19 +43,19 @@ module Ci # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord - def stage_indexes_of_created_builds - created_builds.order(:stage_idx).pluck('distinct stage_idx') + def stage_indexes_of_created_processables + created_processables.order(:stage_idx).pluck('distinct stage_idx') end # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord - def created_builds_in_stage(index) - created_builds.where(stage_idx: index) + def created_processables_in_stage(index) + created_processables.where(stage_idx: index) end # rubocop: enable CodeReuse/ActiveRecord - def created_builds - pipeline.builds.created + def created_processables + pipeline.processables.created end # This method is for compatibility and data consistency and should be removed with 9.3 version of GitLab diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 1d8904f7b29..290c9591b98 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -67,7 +67,7 @@ module Gitlab entry :only, Entry::Policy, description: 'Refs policy this job will be executed for.', - default: { refs: %w[branches tags] } + default: Entry::Policy::DEFAULT_ONLY entry :except, Entry::Policy, description: 'Refs policy this job will be executed for.' diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index 82b72e40404..9845c4af655 100644 --- a/lib/gitlab/ci/config/entry/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -28,11 +28,15 @@ module Gitlab name.to_s.start_with?('.') end + def node_type(name) + hidden?(name) ? Entry::Hidden : Entry::Job + end + # rubocop: disable CodeReuse/ActiveRecord def compose!(deps = nil) super do @config.each do |name, config| - node = hidden?(name) ? Entry::Hidden : Entry::Job + node = node_type(name) factory = ::Gitlab::Config::Entry::Factory.new(node) .value(config || {}) diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 9c677bf6617..adc3660d950 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -11,6 +11,8 @@ module Gitlab strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) } + DEFAULT_ONLY = { refs: %w[branches tags] }.freeze + class RefsPolicy < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index ef738a93bfe..d8296940a04 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -38,9 +38,17 @@ module Gitlab ) end + def bridge? + @attributes.to_h.dig(:options, :trigger).present? + end + def to_resource strong_memoize(:resource) do - ::Ci::Build.new(attributes) + if bridge? + ::Ci::Bridge.new(attributes) + else + ::Ci::Build.new(attributes) + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 4775ff15581..9c15064756a 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -39,7 +39,13 @@ module Gitlab def to_resource strong_memoize(:stage) do ::Ci::Stage.new(attributes).tap do |stage| - seeds.each { |seed| stage.builds << seed.to_resource } + seeds.each do |seed| + if seed.bridge? + stage.bridges << seed.to_resource + else + stage.builds << seed.to_resource + end + end end end end diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index c6cb620f7a0..4746195c618 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -18,7 +18,6 @@ module Gitlab end def details_path - raise NotImplementedError end end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 0c48a6ab3ac..07ba6f83d47 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -33,7 +33,7 @@ module Gitlab { stage_idx: @stages.index(job[:stage]), stage: job[:stage], - tag_list: job[:tags] || [], + tag_list: job[:tags], name: job[:name].to_s, allow_failure: job[:ignore], when: job[:when] || 'on_success', @@ -53,8 +53,9 @@ module Gitlab retry: job[:retry], parallel: job[:parallel], instance: job[:instance], - start_in: job[:start_in] - }.compact } + start_in: job[:start_in], + trigger: job[:trigger] + }.compact }.compact end def stage_builds_attributes(stage) diff --git a/spec/factories/ci/bridge.rb b/spec/factories/ci/bridge.rb index 5f83b80ad7b..39427f416a0 100644 --- a/spec/factories/ci/bridge.rb +++ b/spec/factories/ci/bridge.rb @@ -10,8 +10,16 @@ FactoryBot.define do pipeline factory: :ci_pipeline + transient { downstream nil } + after(:build) do |bridge, evaluator| bridge.project ||= bridge.pipeline.project + + if evaluator.downstream.present? + bridge.options = bridge.options.to_h.merge( + trigger: { project: evaluator.downstream.full_path } + ) + end end end end diff --git a/spec/features/projects/pipelines/pipeline_spec.rb b/spec/features/projects/pipelines/pipeline_spec.rb index 3192c9ffad4..72ef460d315 100644 --- a/spec/features/projects/pipelines/pipeline_spec.rb +++ b/spec/features/projects/pipelines/pipeline_spec.rb @@ -286,6 +286,49 @@ describe 'Pipeline', :js do end end + context 'when a bridge job exists' do + include_context 'pipeline builds' + + let(:project) { create(:project, :repository) } + let(:downstream) { create(:project, :repository) } + + let(:pipeline) do + create(:ci_pipeline, project: project, + ref: 'master', + sha: project.commit.id, + user: user) + end + + let!(:bridge) do + create(:ci_bridge, pipeline: pipeline, + name: 'cross-build', + user: user, + downstream: downstream) + end + + describe 'GET /:project/pipelines/:id' do + before do + visit project_pipeline_path(project, pipeline) + end + + it 'shows the pipeline with a bridge job' do + expect(page).to have_selector('.pipeline-visualization') + expect(page).to have_content('cross-build') + end + end + + describe 'GET /:project/pipelines/:id/builds' do + before do + visit builds_project_pipeline_path(project, pipeline) + end + + it 'shows a bridge job on a list' do + expect(page).to have_content('cross-build') + expect(page).to have_content(bridge.id) + end + end + end + describe 'GET /:project/pipelines/:id/builds' do include_context 'pipeline builds' diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb index 3459939267a..0302e4090cf 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb @@ -163,14 +163,14 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do ->(pipeline) { pipeline.variables.create!(key: 'VAR', value: '123') } end - it 'raises exception' do + it 'wastes pipeline iid' do expect { step.perform! }.to raise_error(ActiveRecord::RecordNotSaved) - end - it 'wastes pipeline iid' do - expect { step.perform! }.to raise_error + last_iid = InternalId.ci_pipelines + .where(project_id: project.id) + .last.last_value - expect(InternalId.ci_pipelines.where(project_id: project.id).last.last_value).to be > 0 + expect(last_iid).to be > 0 end end end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index a700cfd4546..fae8add6453 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -5,8 +5,7 @@ describe Gitlab::Ci::Pipeline::Seed::Build do let(:pipeline) { create(:ci_empty_pipeline, project: project) } let(:attributes) do - { name: 'rspec', - ref: 'master' } + { name: 'rspec', ref: 'master' } end subject do @@ -21,10 +20,45 @@ describe Gitlab::Ci::Pipeline::Seed::Build do end end + describe '#bridge?' do + context 'when job is a bridge' do + let(:attributes) do + { name: 'rspec', ref: 'master', options: { trigger: 'my/project' } } + end + + it { is_expected.to be_bridge } + end + + context 'when trigger definition is empty' do + let(:attributes) do + { name: 'rspec', ref: 'master', options: { trigger: '' } } + end + + it { is_expected.not_to be_bridge } + end + + context 'when job is not a bridge' do + it { is_expected.not_to be_bridge } + end + end + describe '#to_resource' do - it 'returns a valid build resource' do - expect(subject.to_resource).to be_a(::Ci::Build) - expect(subject.to_resource).to be_valid + context 'when job is not a bridge' do + it 'returns a valid build resource' do + expect(subject.to_resource).to be_a(::Ci::Build) + expect(subject.to_resource).to be_valid + end + end + + context 'when job is a bridge' do + let(:attributes) do + { name: 'rspec', ref: 'master', options: { trigger: 'my/project' } } + end + + it 'returns a valid bridge resource' do + expect(subject.to_resource).to be_a(::Ci::Bridge) + expect(subject.to_resource).to be_valid + end end it 'memoizes a resource object' do diff --git a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb index 82f741845db..493ca3cd7b5 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb @@ -62,8 +62,18 @@ describe Gitlab::Ci::Pipeline::Seed::Stage do expect(subject.seeds.map(&:attributes)).to all(include(ref: 'master')) expect(subject.seeds.map(&:attributes)).to all(include(tag: false)) expect(subject.seeds.map(&:attributes)).to all(include(project: pipeline.project)) - expect(subject.seeds.map(&:attributes)) - .to all(include(trigger_request: pipeline.trigger_requests.first)) + end + + context 'when a legacy trigger exists' do + before do + create(:ci_trigger_request, pipeline: pipeline) + end + + it 'returns build seeds including legacy trigger' do + expect(pipeline.legacy_trigger).not_to be_nil + expect(subject.seeds.map(&:attributes)) + .to all(include(trigger_request: pipeline.legacy_trigger)) + end end context 'when a ref is protected' do diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 20c35573cfb..91139d421f5 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -21,15 +21,12 @@ module Gitlab stage: "test", stage_idx: 1, name: "rspec", - coverage_regex: nil, - tag_list: [], options: { before_script: ["pwd"], script: ["rspec"] }, allow_failure: false, when: "on_success", - environment: nil, yaml_variables: [] }) end @@ -154,12 +151,9 @@ module Gitlab builds: [{ stage_idx: 1, stage: "test", - tag_list: [], name: "rspec", allow_failure: false, when: "on_success", - environment: nil, - coverage_regex: nil, yaml_variables: [], options: { script: ["rspec"] }, only: { refs: ["branches"] }, @@ -169,12 +163,9 @@ module Gitlab builds: [{ stage_idx: 2, stage: "deploy", - tag_list: [], name: "prod", allow_failure: false, when: "on_success", - environment: nil, - coverage_regex: nil, yaml_variables: [], options: { script: ["cap prod"] }, only: { refs: ["tags"] }, @@ -344,8 +335,6 @@ module Gitlab stage: "test", stage_idx: 1, name: "rspec", - coverage_regex: nil, - tag_list: [], options: { before_script: ["pwd"], script: ["rspec"], @@ -356,7 +345,6 @@ module Gitlab }, allow_failure: false, when: "on_success", - environment: nil, yaml_variables: [] }) end @@ -378,8 +366,6 @@ module Gitlab stage: "test", stage_idx: 1, name: "rspec", - coverage_regex: nil, - tag_list: [], options: { before_script: ["pwd"], script: ["rspec"], @@ -390,7 +376,6 @@ module Gitlab }, allow_failure: false, when: "on_success", - environment: nil, yaml_variables: [] }) end @@ -410,8 +395,6 @@ module Gitlab stage: "test", stage_idx: 1, name: "rspec", - coverage_regex: nil, - tag_list: [], options: { before_script: ["pwd"], script: ["rspec"], @@ -420,7 +403,6 @@ module Gitlab }, allow_failure: false, when: "on_success", - environment: nil, yaml_variables: [] }) end @@ -438,8 +420,6 @@ module Gitlab stage: "test", stage_idx: 1, name: "rspec", - coverage_regex: nil, - tag_list: [], options: { before_script: ["pwd"], script: ["rspec"], @@ -448,7 +428,6 @@ module Gitlab }, allow_failure: false, when: "on_success", - environment: nil, yaml_variables: [] }) end @@ -763,8 +742,6 @@ module Gitlab stage: "test", stage_idx: 1, name: "rspec", - coverage_regex: nil, - tag_list: [], options: { before_script: ["pwd"], script: ["rspec"], @@ -779,7 +756,6 @@ module Gitlab }, when: "on_success", allow_failure: false, - environment: nil, yaml_variables: [] }) end @@ -976,14 +952,11 @@ module Gitlab stage: "test", stage_idx: 1, name: "normal_job", - coverage_regex: nil, - tag_list: [], options: { script: ["test"] }, when: "on_success", allow_failure: false, - environment: nil, yaml_variables: [] }) end @@ -1023,28 +996,22 @@ module Gitlab stage: "build", stage_idx: 0, name: "job1", - coverage_regex: nil, - tag_list: [], options: { script: ["execute-script-for-job"] }, when: "on_success", allow_failure: false, - environment: nil, yaml_variables: [] }) expect(subject.second).to eq({ stage: "build", stage_idx: 0, name: "job2", - coverage_regex: nil, - tag_list: [], options: { script: ["execute-script-for-job"] }, when: "on_success", allow_failure: false, - environment: nil, yaml_variables: [] }) end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 5afa9669b1a..6897ac8a3a8 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -114,6 +114,7 @@ ci_pipelines: - stages - statuses - builds +- processables - trigger_requests - variables - auto_canceled_by @@ -137,6 +138,7 @@ stages: - pipeline - statuses - builds +- bridges statuses: - project - pipeline diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index 17f33785fda..72a0df96a80 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -39,6 +39,29 @@ describe Ci::Pipeline, :mailer do end end + describe '.processables' do + before do + create(:ci_build, name: 'build', pipeline: pipeline) + create(:ci_bridge, name: 'bridge', pipeline: pipeline) + create(:commit_status, name: 'commit status', pipeline: pipeline) + create(:generic_commit_status, name: 'generic status', pipeline: pipeline) + end + + it 'has an association with processable CI/CD entities' do + pipeline.processables.pluck('name').yield_self do |processables| + expect(processables).to match_array %w[build bridge] + end + end + + it 'makes it possible to append a new processable' do + pipeline.processables << build(:ci_bridge) + + pipeline.save! + + expect(pipeline.processables.reload.count).to eq 3 + end + end + describe '.sort_by_merge_request_pipelines' do subject { described_class.sort_by_merge_request_pipelines } |