diff options
23 files changed, 116 insertions, 30 deletions
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 53af87a271a..5eb30f4aaa0 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -7,12 +7,17 @@ module Ci include Presentable include Gitlab::OptimisticLocking include Gitlab::Utils::StrongMemoize + include AtomicInternalId belongs_to :project, inverse_of: :pipelines belongs_to :user belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' + has_internal_id :iid, scope: :project, presence: false, init: ->(s) do + s&.project&.pipelines&.maximum(:iid) || s&.project&.pipelines&.count + end + has_many :stages has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline @@ -531,6 +536,7 @@ module Ci def predefined_variables Gitlab::Ci::Variables::Collection.new + .append(key: 'CI_PIPELINE_IID', value: iid.to_s) .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path) .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) .append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message) diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 22f516a172f..164c704260e 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -25,9 +25,13 @@ module AtomicInternalId extend ActiveSupport::Concern module ClassMethods - def has_internal_id(column, scope:, init:) # rubocop:disable Naming/PredicateName - before_validation(on: :create) do + def has_internal_id(column, scope:, init:, presence: true) # rubocop:disable Naming/PredicateName + before_validation :"ensure_#{scope}_#{column}!", on: :create + validates column, presence: presence + + define_method("ensure_#{scope}_#{column}!") do scope_value = association(scope).reader + if read_attribute(column).blank? && scope_value scope_attrs = { scope_value.class.table_name.singularize.to_sym => scope_value } usage = self.class.table_name.to_sym @@ -35,13 +39,9 @@ module AtomicInternalId new_iid = InternalId.generate_next(self, scope_attrs, usage, init) write_attribute(column, new_iid) end - end - validates column, presence: true, numericality: true + read_attribute(column) + end end end - - def to_param - iid.to_s - end end diff --git a/app/models/concerns/iid_routes.rb b/app/models/concerns/iid_routes.rb new file mode 100644 index 00000000000..246748cf52c --- /dev/null +++ b/app/models/concerns/iid_routes.rb @@ -0,0 +1,9 @@ +module IidRoutes + ## + # This automagically enforces all related routes to use `iid` instead of `id` + # If you want to use `iid` for some routes and `id` for other routes, this module should not to be included, + # instead you should define `iid` or `id` explictly at each route generators. e.g. pipeline_path(project.id, pipeline.iid) + def to_param + iid.to_s + end +end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 254764eefde..ac86e9e8de0 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -1,5 +1,6 @@ class Deployment < ActiveRecord::Base include AtomicInternalId + include IidRoutes belongs_to :project, required: true belongs_to :environment, required: true diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb index f7f930e86ed..f50f28deffe 100644 --- a/app/models/internal_id.rb +++ b/app/models/internal_id.rb @@ -14,7 +14,7 @@ class InternalId < ActiveRecord::Base belongs_to :project belongs_to :namespace - enum usage: { issues: 0, merge_requests: 1, deployments: 2, milestones: 3, epics: 4 } + enum usage: { issues: 0, merge_requests: 1, deployments: 2, milestones: 3, epics: 4, ci_pipelines: 5 } validates :usage, presence: true diff --git a/app/models/issue.rb b/app/models/issue.rb index 0332bfa9371..d0e8fe908e4 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -2,6 +2,7 @@ require 'carrierwave/orm/activerecord' class Issue < ActiveRecord::Base include AtomicInternalId + include IidRoutes include Issuable include Noteable include Referable diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 9c4384a6e42..a00b8d4e2c1 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1,5 +1,6 @@ class MergeRequest < ActiveRecord::Base include AtomicInternalId + include IidRoutes include Issuable include Noteable include Referable diff --git a/app/models/milestone.rb b/app/models/milestone.rb index d14e3a4ded5..d05dcfd083a 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -9,6 +9,7 @@ class Milestone < ActiveRecord::Base include CacheMarkdownField include AtomicInternalId + include IidRoutes include Sortable include Referable include StripAttribute diff --git a/changelogs/unreleased/per-project-pipeline-iid.yml b/changelogs/unreleased/per-project-pipeline-iid.yml new file mode 100644 index 00000000000..78a513a9986 --- /dev/null +++ b/changelogs/unreleased/per-project-pipeline-iid.yml @@ -0,0 +1,5 @@ +--- +title: Add per-project pipeline id +merge_request: 18558 +author: +type: added diff --git a/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb b/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb new file mode 100644 index 00000000000..e8f0c91d612 --- /dev/null +++ b/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb @@ -0,0 +1,13 @@ +class AddPipelineIidToCiPipelines < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + add_column :ci_pipelines, :iid, :integer + end + + def down + remove_column :ci_pipelines, :iid, :integer + end +end diff --git a/db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb b/db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb new file mode 100644 index 00000000000..3fa59b44d5d --- /dev/null +++ b/db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb @@ -0,0 +1,15 @@ +class AddIndexConstraintsToPipelineIid < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_pipelines, [:project_id, :iid], unique: true, where: 'iid IS NOT NULL' + end + + def down + remove_concurrent_index :ci_pipelines, [:project_id, :iid] + end +end diff --git a/db/schema.rb b/db/schema.rb index 932b7f8da02..359d25dafb5 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -451,10 +451,12 @@ ActiveRecord::Schema.define(version: 20180529093006) do t.integer "config_source" t.boolean "protected" t.integer "failure_reason" + t.integer "iid" end add_index "ci_pipelines", ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id", using: :btree add_index "ci_pipelines", ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id", using: :btree + add_index "ci_pipelines", ["project_id", "iid"], name: "index_ci_pipelines_on_project_id_and_iid", unique: true, where: "(iid IS NOT NULL)", using: :btree add_index "ci_pipelines", ["project_id", "ref", "status", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_status_and_id", using: :btree add_index "ci_pipelines", ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree add_index "ci_pipelines", ["project_id"], name: "index_ci_pipelines_on_project_id", using: :btree diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index f10423b92cf..dfea10314b9 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -72,6 +72,7 @@ future GitLab releases.** | **CI_RUNNER_REVISION** | all | 10.6 | GitLab Runner revision that is executing the current job | | **CI_RUNNER_EXECUTABLE_ARCH** | all | 10.6 | The OS/architecture of the GitLab Runner executable (note that this is not necessarily the same as the environment of the executor) | | **CI_PIPELINE_ID** | 8.10 | 0.5 | The unique id of the current pipeline that GitLab CI uses internally | +| **CI_PIPELINE_IID** | 11.0 | all | The unique id of the current pipeline scoped to project | | **CI_PIPELINE_TRIGGERED** | all | all | The flag to indicate that job was [triggered] | | **CI_PIPELINE_SOURCE** | 10.0 | all | Indicates how the pipeline was triggered. Possible options are: `push`, `web`, `trigger`, `schedule`, `api`, and `pipeline`. For pipelines created before GitLab 9.5, this will show as `unknown` | | **CI_PROJECT_DIR** | all | all | The full path where the repository is cloned and where the job is run | diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index f4c8d5342c1..918a0d151fc 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -6,6 +6,9 @@ module Gitlab include Chain::Helpers def perform! + # Allocate next IID outside of transaction + pipeline.ensure_project_iid! + ::Ci::Pipeline.transaction do pipeline.save! diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb index 0edc3f315bb..de69a65aee4 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb @@ -37,21 +37,23 @@ describe Gitlab::Ci::Pipeline::Chain::Create do end context 'when pipeline has validation errors' do - let(:pipeline) do - build(:ci_pipeline, project: project, ref: nil) - end - - before do - step.perform! - end - - it 'breaks the chain' do - expect(step.break?).to be true - end - - it 'appends validation error' do - expect(pipeline.errors.to_a) - .to include /Failed to persist the pipeline/ + context 'when ref is nil' do + let(:pipeline) do + build(:ci_pipeline, project: project, ref: nil) + end + + before do + step.perform! + end + + it 'breaks the chain' do + expect(step.break?).to be true + end + + it 'appends validation error' do + expect(pipeline.errors.to_a) + .to include /Failed to persist the pipeline/ + end end end end diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 62da967cf96..2619c6a10d8 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -228,6 +228,7 @@ Ci::Pipeline: - config_source - failure_reason - protected +- iid Ci::Stage: - id - name diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 77179028ede..c89655f4058 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1565,6 +1565,7 @@ describe Ci::Build do { key: 'CI_PROJECT_NAMESPACE', value: project.namespace.full_path, public: true }, { key: 'CI_PROJECT_URL', value: project.web_url, public: true }, { key: 'CI_PROJECT_VISIBILITY', value: 'private', public: true }, + { key: 'CI_PIPELINE_IID', value: pipeline.iid.to_s, public: true }, { key: 'CI_CONFIG_PATH', value: pipeline.ci_yaml_file_path, public: true }, { key: 'CI_PIPELINE_SOURCE', value: pipeline.source, public: true }, { key: 'CI_COMMIT_MESSAGE', value: pipeline.git_commit_message, public: true }, diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index e4f4c62bd22..314cb3a28ed 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -35,6 +35,17 @@ describe Ci::Pipeline, :mailer do end end + describe 'modules' do + it_behaves_like 'AtomicInternalId' do + let(:internal_id_attribute) { :iid } + let(:instance) { build(:ci_pipeline) } + let(:scope) { :project } + let(:scope_attrs) { { project: instance.project } } + let(:usage) { :ci_pipelines } + let(:allow_nil) { true } + end + end + describe '#source' do context 'when creating new pipeline' do let(:pipeline) do @@ -195,7 +206,8 @@ describe Ci::Pipeline, :mailer do it 'includes all predefined variables in a valid order' do keys = subject.map { |variable| variable[:key] } - expect(keys).to eq %w[CI_CONFIG_PATH + expect(keys).to eq %w[CI_PIPELINE_IID + CI_CONFIG_PATH CI_PIPELINE_SOURCE CI_COMMIT_MESSAGE CI_COMMIT_TITLE diff --git a/spec/models/deployment_spec.rb b/spec/models/deployment_spec.rb index aee70bcfb29..e01906f4b6c 100644 --- a/spec/models/deployment_spec.rb +++ b/spec/models/deployment_spec.rb @@ -20,6 +20,7 @@ describe Deployment do it_behaves_like 'AtomicInternalId' do let(:internal_id_attribute) { :iid } let(:instance) { build(:deployment) } + let(:scope) { :project } let(:scope_attrs) { { project: instance.project } } let(:usage) { :deployments } end diff --git a/spec/models/issue_spec.rb b/spec/models/issue_spec.rb index 128acf83686..e818fbeb9cf 100644 --- a/spec/models/issue_spec.rb +++ b/spec/models/issue_spec.rb @@ -17,6 +17,7 @@ describe Issue do it_behaves_like 'AtomicInternalId' do let(:internal_id_attribute) { :iid } let(:instance) { build(:issue) } + let(:scope) { :project } let(:scope_attrs) { { project: instance.project } } let(:usage) { :issues } end diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index 92e33a64d26..aa968dda41f 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -25,6 +25,7 @@ describe MergeRequest do it_behaves_like 'AtomicInternalId' do let(:internal_id_attribute) { :iid } let(:instance) { build(:merge_request) } + let(:scope) { :target_project } let(:scope_attrs) { { project: instance.target_project } } let(:usage) { :merge_requests } end diff --git a/spec/models/milestone_spec.rb b/spec/models/milestone_spec.rb index 4bb9717d33e..204d6b47832 100644 --- a/spec/models/milestone_spec.rb +++ b/spec/models/milestone_spec.rb @@ -6,6 +6,7 @@ describe Milestone do it_behaves_like 'AtomicInternalId' do let(:internal_id_attribute) { :iid } let(:instance) { build(:milestone, project: build(:project), group: nil) } + let(:scope) { :project } let(:scope_attrs) { { project: instance.project } } let(:usage) { :milestones } end @@ -15,6 +16,7 @@ describe Milestone do it_behaves_like 'AtomicInternalId' do let(:internal_id_attribute) { :iid } let(:instance) { build(:milestone, project: nil, group: build(:group)) } + let(:scope) { :group } let(:scope_attrs) { { namespace: instance.group } } let(:usage) { :milestones } end diff --git a/spec/support/shared_examples/models/atomic_internal_id_spec.rb b/spec/support/shared_examples/models/atomic_internal_id_spec.rb index 6a6e13418a9..a05279364f2 100644 --- a/spec/support/shared_examples/models/atomic_internal_id_spec.rb +++ b/spec/support/shared_examples/models/atomic_internal_id_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' shared_examples_for 'AtomicInternalId' do + let(:allow_nil) { false } + describe '.has_internal_id' do describe 'Module inclusion' do subject { described_class } @@ -9,14 +11,19 @@ shared_examples_for 'AtomicInternalId' do end describe 'Validation' do - subject { instance } - before do - allow(InternalId).to receive(:generate_next).and_return(nil) + allow_any_instance_of(described_class).to receive(:"ensure_#{scope}_#{internal_id_attribute}!") {} end - it { is_expected.to validate_presence_of(internal_id_attribute) } - it { is_expected.to validate_numericality_of(internal_id_attribute) } + it 'validates presence' do + instance.valid? + + if allow_nil + expect(instance.errors[internal_id_attribute]).to be_empty + else + expect(instance.errors[internal_id_attribute]).to include("can't be blank") + end + end end describe 'Creating an instance' do |