From b21ee2ee36e1aaddbe0b3541a8cac5f117143b66 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 11:29:04 +0200 Subject: Add initial stage_id background migration files --- ...858_migrate_stage_id_reference_in_background.rb | 12 ++++++++++ .../migrate_build_stage_id_reference.rb | 16 +++++++++++++ ...igrate_stage_id_reference_in_background_spec.rb | 26 ++++++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb create mode 100644 lib/gitlab/background_migration/migrate_build_stage_id_reference.rb create mode 100644 spec/migrations/migrate_stage_id_reference_in_background_spec.rb diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb new file mode 100644 index 00000000000..2eaa798d0aa --- /dev/null +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -0,0 +1,12 @@ +class MigrateStageIdReferenceInBackground < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + end + + def down + # noop + end +end diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb new file mode 100644 index 00000000000..b554c3e079b --- /dev/null +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -0,0 +1,16 @@ +module Gitlab + module BackgroundMigration + class MigrateBuildStageIdReference + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + end + + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' + end + + def perform(id) + end + end + end +end diff --git a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb new file mode 100644 index 00000000000..f86ef834afa --- /dev/null +++ b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb @@ -0,0 +1,26 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20170628080858_migrate_stage_id_reference_in_background') + +describe MigrateStageIdReferenceInBackground, :migration, :redis do + let(:jobs) { table(:ci_builds) } + let(:stages) { table(:ci_stages) } + let(:pipelines) { table(:ci_pipelines) } + let(:projects) { table(:projects) } + + before do + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') + + jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test') + jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy') + + stages.create(id: 101, pipeline_id: 1, project_id: 123, name: 'test') + stages.create(id: 102, pipeline_id: 1, project_id: 123, name: 'build') + stages.create(id: 103, pipeline_id: 1, project_id: 123, name: 'deploy') + end + + it 'schedules background migrations' do + end +end -- cgit v1.2.1 From 98992c4e4b3231a99eb5ff17c44e96fe79a6cff2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 12:01:52 +0200 Subject: Add initial build stage_id ref background migration --- ...8080858_migrate_stage_id_reference_in_background.rb | 10 ++++++++++ .../migrate_build_stage_id_reference.rb | 18 +++++++++++------- .../migrate_stage_id_reference_in_background_spec.rb | 5 +++++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 2eaa798d0aa..44bac4a8cc7 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -3,7 +3,17 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration DOWNTIME = false + disable_ddl_transaction! + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + end + def up + Build.find_each do |build| + BackgroundMigrationWorker + .perform_async('MigrateBuildStageIdReference', [build.id]) + end end def down diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb index b554c3e079b..87c6c4ed49f 100644 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -1,15 +1,19 @@ module Gitlab module BackgroundMigration class MigrateBuildStageIdReference - class Build < ActiveRecord::Base - self.table_name = 'ci_builds' - end + def perform(id) + raise ArgumentError unless id.is_a?(Integer) - class Stage < ActiveRecord::Base - self.table_name = 'ci_stages' - end + sql = <<-SQL.strip_heredoc + UPDATE "ci_builds" SET "stage_id" = ( + SELECT id FROM ci_stages + WHERE ci_stages.pipeline_id = ci_builds.commit_id + AND ci_stages.name = ci_builds.stage + ) + WHERE "ci_builds"."id" = #{id} AND "ci_builds"."stage_id" IS NULL + SQL - def perform(id) + ActiveRecord::Base.connection.execute(sql) end end end diff --git a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb index f86ef834afa..ea3a18802d9 100644 --- a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb +++ b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb @@ -22,5 +22,10 @@ describe MigrateStageIdReferenceInBackground, :migration, :redis do end it 'schedules background migrations' do + expect(jobs.where(stage_id: nil)).to be_present + + migrate! + + expect(jobs.where(stage_id: nil)).to be_empty end end -- cgit v1.2.1 From 6209ff671fdd025be31f9dcaf208a71b6ec2907d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 12:20:45 +0200 Subject: Update `db/schema.rb` with a new schema version --- db/schema.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index 8c7440ee610..3a45e0fe562 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170622162730) do +ActiveRecord::Schema.define(version: 20170628080858) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" -- cgit v1.2.1 From 02bb40e2acd7b1838e47e1a2f8b9288e42e6ca53 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 12:21:25 +0200 Subject: Find builds that require a migration in batches --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 9 ++++++--- spec/migrations/migrate_stage_id_reference_in_background_spec.rb | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 44bac4a8cc7..6b326bc0b69 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -2,6 +2,8 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers DOWNTIME = false + BATCH_SIZE = 10000 + MIGRATION = 'MigrateBuildStageIdReference'.freeze disable_ddl_transaction! @@ -10,9 +12,10 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration end def up - Build.find_each do |build| - BackgroundMigrationWorker - .perform_async('MigrateBuildStageIdReference', [build.id]) + Build.find_in_batches(batch_size: BATCH_SIZE).with_index do |builds, batch| + migrations = builds.map { |build| [MIGRATION, [build.id]] } + + BackgroundMigrationWorker.perform_bulk(*migrations) end end diff --git a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb index ea3a18802d9..d515eb42b9d 100644 --- a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb +++ b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb @@ -8,6 +8,8 @@ describe MigrateStageIdReferenceInBackground, :migration, :redis do let(:projects) { table(:projects) } before do + stub_const('MigrateStageIdReferenceInBackground::BATCH_SIZE', 1) + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1') pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') -- cgit v1.2.1 From 5292eb651e1e3595e409a4c216eb0be3445a9319 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 12:23:00 +0200 Subject: Schedule background migration only when it is needed --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 6b326bc0b69..bfeb09f6da1 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -12,11 +12,13 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration end def up - Build.find_in_batches(batch_size: BATCH_SIZE).with_index do |builds, batch| - migrations = builds.map { |build| [MIGRATION, [build.id]] } + Build.where(stage_id: nil) + .find_in_batches(batch_size: BATCH_SIZE) + .with_index do |builds, batch| + migrations = builds.map { |build| [MIGRATION, [build.id]] } - BackgroundMigrationWorker.perform_bulk(*migrations) - end + BackgroundMigrationWorker.perform_bulk(*migrations) + end end def down -- cgit v1.2.1 From 187dd1005cd92c530146d7f5b0a89b368b09c3e9 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 15:24:53 +0200 Subject: Add specs for delayed stage_id background migrations --- ...858_migrate_stage_id_reference_in_background.rb | 7 +-- ...igrate_stage_id_reference_in_background_spec.rb | 51 ++++++++++++++++++++-- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index bfeb09f6da1..a73456af386 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -15,9 +15,10 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration Build.where(stage_id: nil) .find_in_batches(batch_size: BATCH_SIZE) .with_index do |builds, batch| - migrations = builds.map { |build| [MIGRATION, [build.id]] } - - BackgroundMigrationWorker.perform_bulk(*migrations) + builds.each do |build| + schedule = (batch - 1) * 5.minutes + BackgroundMigrationWorker.perform_at(schedule, MIGRATION, [build.id]) + end end end diff --git a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb index d515eb42b9d..8b497656377 100644 --- a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb +++ b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb @@ -1,7 +1,37 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20170628080858_migrate_stage_id_reference_in_background') -describe MigrateStageIdReferenceInBackground, :migration, :redis do +RSpec::Matchers.define :have_migrated do |*expected| + match do |migration| + BackgroundMigrationWorker.jobs.any? do |job| + job['enqueued_at'].present? && job['args'] == [migration, expected] + end + end + + failure_message do |migration| + <<-EOS + Background migration `#{migration}` with args `#{expected.inspect}` + not migrated! + EOS + end +end + +RSpec::Matchers.define :have_scheduled_migration do |time, *expected| + match do |migration| + BackgroundMigrationWorker.jobs.any? do |job| + job['args'] == [migration, expected] && job['at'] >= time + end + end + + failure_message do |migration| + <<-EOS + Background migration `#{migration}` with args `#{expected.inspect}` + not scheduled! + EOS + end +end + +describe MigrateStageIdReferenceInBackground, :migration do let(:jobs) { table(:ci_builds) } let(:stages) { table(:ci_stages) } let(:pipelines) { table(:ci_pipelines) } @@ -23,11 +53,24 @@ describe MigrateStageIdReferenceInBackground, :migration, :redis do stages.create(id: 103, pipeline_id: 1, project_id: 123, name: 'deploy') end + it 'correctly schedules background migrations' do + Sidekiq::Testing.fake! do + migrate! + + expect(described_class::MIGRATION).to have_migrated(1) + expect(described_class::MIGRATION).to have_migrated(2) + expect(described_class::MIGRATION).to have_scheduled_migration(5.minutes, 3) + expect(described_class::MIGRATION).to have_scheduled_migration(5.minutes, 4) + end + end + it 'schedules background migrations' do - expect(jobs.where(stage_id: nil)).to be_present + Sidekiq::Testing.inline! do + expect(jobs.where(stage_id: nil)).to be_present - migrate! + migrate! - expect(jobs.where(stage_id: nil)).to be_empty + expect(jobs.where(stage_id: nil)).to be_empty + end end end -- cgit v1.2.1 From af2f2dc5ed588d33919d5db3f684c165d7427ab7 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 11:41:19 +0200 Subject: Make it possible to schedule bg migrations in bulk --- app/workers/background_migration_worker.rb | 18 +++++++++++-- doc/development/background_migrations.md | 19 +++++++++++--- spec/support/sidekiq.rb | 8 +++++- spec/workers/background_migration_worker_spec.rb | 33 +++++++++++++++++++++++- 4 files changed, 70 insertions(+), 8 deletions(-) diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index e85e221d353..751f37a3c39 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -2,18 +2,32 @@ class BackgroundMigrationWorker include Sidekiq::Worker include DedicatedSidekiqQueue - # Schedules a number of jobs in bulk + # Enqueues a number of jobs in bulk. # # The `jobs` argument should be an Array of Arrays, each sub-array must be in # the form: # # [migration-class, [arg1, arg2, ...]] - def self.perform_bulk(*jobs) + def self.perform_bulk(jobs) Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], 'args' => jobs) end + # Schedules a number of jobs in bulk, with a delay. + # + def self.perform_bulk_in(delay, jobs) + now = Time.now.to_f + schedule = now + delay.to_f + + raise ArgumentError if schedule <= now + + Sidekiq::Client.push_bulk('class' => self, + 'queue' => sidekiq_options['queue'], + 'args' => jobs, + 'at' => schedule) + end + # Performs the background migration. # # See Gitlab::BackgroundMigration.perform for more information. diff --git a/doc/development/background_migrations.md b/doc/development/background_migrations.md index 0239e6b3163..a58f161fc30 100644 --- a/doc/development/background_migrations.md +++ b/doc/development/background_migrations.md @@ -50,14 +50,14 @@ your migration: BackgroundMigrationWorker.perform_async('BackgroundMigrationClassName', [arg1, arg2, ...]) ``` -Usually it's better to schedule jobs in bulk, for this you can use +Usually it's better to enqueue jobs in bulk, for this you can use `BackgroundMigrationWorker.perform_bulk`: ```ruby BackgroundMigrationWorker.perform_bulk( - ['BackgroundMigrationClassName', [1]], - ['BackgroundMigrationClassName', [2]], - ... + [['BackgroundMigrationClassName', [1]], + ['BackgroundMigrationClassName', [2]], + ...] ) ``` @@ -68,6 +68,17 @@ consuming migrations it's best to schedule a background job using an updates. Removals in turn can be handled by simply defining foreign keys with cascading deletes. +If you would like to schedule jobs in bulk with a delay, you can use +`BackgroundMigrationWorker.perform_bulk_in`: + +```ruby +jobs = [['BackgroundMigrationClassName', [1]], + ['BackgroundMigrationClassName', [2]], + ...] + +BackgroundMigrationWorker.perform_bulk_in(5.minutes, jobs) +``` + ## Cleaning Up Because background migrations can take a long time you can't immediately clean diff --git a/spec/support/sidekiq.rb b/spec/support/sidekiq.rb index 575d3451150..f3819ed2353 100644 --- a/spec/support/sidekiq.rb +++ b/spec/support/sidekiq.rb @@ -1,5 +1,11 @@ -require 'sidekiq/testing/inline' +require 'sidekiq/testing' Sidekiq::Testing.server_middleware do |chain| chain.add Gitlab::SidekiqStatus::ServerMiddleware end + +RSpec.configure do |config| + config.after(:each, :sidekiq) do + Sidekiq::Worker.clear_all + end +end diff --git a/spec/workers/background_migration_worker_spec.rb b/spec/workers/background_migration_worker_spec.rb index 85939429feb..4f6e3474634 100644 --- a/spec/workers/background_migration_worker_spec.rb +++ b/spec/workers/background_migration_worker_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe BackgroundMigrationWorker do +describe BackgroundMigrationWorker, :sidekiq do describe '.perform' do it 'performs a background migration' do expect(Gitlab::BackgroundMigration) @@ -10,4 +10,35 @@ describe BackgroundMigrationWorker do described_class.new.perform('Foo', [10, 20]) end end + + describe '.perform_bulk' do + it 'enqueues background migrations in bulk' do + Sidekiq::Testing.fake! do + described_class.perform_bulk([['Foo', [1]], ['Foo', [2]]]) + + expect(described_class.jobs.count).to eq 2 + expect(described_class.jobs).to all(include('enqueued_at')) + end + end + end + + describe '.perform_bulk_in' do + context 'when delay is valid' do + it 'correctly schedules background migrations' do + Sidekiq::Testing.fake! do + described_class.perform_bulk_in(1.minute, [['Foo', [1]], ['Foo', [2]]]) + + expect(described_class.jobs.count).to eq 2 + expect(described_class.jobs).to all(include('at')) + end + end + end + + context 'when delay is invalid' do + it 'raises an ArgumentError exception' do + expect { described_class.perform_bulk_in(-60, [['Foo']]) } + .to raise_error(ArgumentError) + end + end + end end -- cgit v1.2.1 From 69736f3927160bd362e165b4cd9e78912a3c30c0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 12:10:29 +0200 Subject: Use ActiveRecord 5 batching to schedule bg migration --- app/workers/background_migration_worker.rb | 2 +- config/initializers/ar5_batching.rb | 4 +- ...858_migrate_stage_id_reference_in_background.rb | 7 ++- ...igrate_stage_id_reference_in_background_spec.rb | 55 ++++++++++------------ 4 files changed, 32 insertions(+), 36 deletions(-) diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index 751f37a3c39..23c297de8bc 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -14,7 +14,7 @@ class BackgroundMigrationWorker 'args' => jobs) end - # Schedules a number of jobs in bulk, with a delay. + # Schedules multiple jobs in bulk, with a delay. # def self.perform_bulk_in(delay, jobs) now = Time.now.to_f diff --git a/config/initializers/ar5_batching.rb b/config/initializers/ar5_batching.rb index 35e8b3808e2..31efef83a6f 100644 --- a/config/initializers/ar5_batching.rb +++ b/config/initializers/ar5_batching.rb @@ -15,7 +15,7 @@ module ActiveRecord relation = relation.where(arel_table[primary_key].lteq(finish)) if finish batch_relation = relation - loop do + 1.step do |index| if load records = batch_relation.records ids = records.map(&:id) @@ -31,7 +31,7 @@ module ActiveRecord primary_key_offset = ids.last raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset - yield yielded_relation + yield yielded_relation, index break if ids.length < of batch_relation = relation.where(arel_table[primary_key].gt(primary_key_offset)) diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index a73456af386..9e95216b35a 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -13,10 +13,9 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration def up Build.where(stage_id: nil) - .find_in_batches(batch_size: BATCH_SIZE) - .with_index do |builds, batch| - builds.each do |build| - schedule = (batch - 1) * 5.minutes + .in_batches(of: BATCH_SIZE) do |relation, index| + schedule = index * 5.minutes + relation.each do |build| BackgroundMigrationWorker.perform_at(schedule, MIGRATION, [build.id]) end end diff --git a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb index 8b497656377..d3645ec0395 100644 --- a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb +++ b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb @@ -1,44 +1,39 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20170628080858_migrate_stage_id_reference_in_background') -RSpec::Matchers.define :have_migrated do |*expected| - match do |migration| - BackgroundMigrationWorker.jobs.any? do |job| - job['enqueued_at'].present? && job['args'] == [migration, expected] +describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do + matcher :have_migrated do |*expected| + match do |migration| + BackgroundMigrationWorker.jobs.any? do |job| + job['enqueued_at'].present? && job['args'] == [migration, expected] + end end - end - failure_message do |migration| - <<-EOS - Background migration `#{migration}` with args `#{expected.inspect}` - not migrated! - EOS + failure_message do |migration| + "Migration `#{migration}` with args `#{expected.inspect}` not executed!" + end end -end -RSpec::Matchers.define :have_scheduled_migration do |time, *expected| - match do |migration| - BackgroundMigrationWorker.jobs.any? do |job| - job['args'] == [migration, expected] && job['at'] >= time + matcher :have_scheduled_migration do |delay, *expected| + match do |migration| + BackgroundMigrationWorker.jobs.any? do |job| + job['args'] == [migration, expected] && + job['at'].to_f == (delay.to_f + Time.now.to_f) + end end - end - failure_message do |migration| - <<-EOS - Background migration `#{migration}` with args `#{expected.inspect}` - not scheduled! - EOS + failure_message do |migration| + "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!" + end end -end -describe MigrateStageIdReferenceInBackground, :migration do let(:jobs) { table(:ci_builds) } let(:stages) { table(:ci_stages) } let(:pipelines) { table(:ci_pipelines) } let(:projects) { table(:projects) } before do - stub_const('MigrateStageIdReferenceInBackground::BATCH_SIZE', 1) + stub_const('MigrateStageIdReferenceInBackground::BATCH_SIZE', 2) projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1') pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') @@ -55,12 +50,14 @@ describe MigrateStageIdReferenceInBackground, :migration do it 'correctly schedules background migrations' do Sidekiq::Testing.fake! do - migrate! + Timecop.freeze do + migrate! - expect(described_class::MIGRATION).to have_migrated(1) - expect(described_class::MIGRATION).to have_migrated(2) - expect(described_class::MIGRATION).to have_scheduled_migration(5.minutes, 3) - expect(described_class::MIGRATION).to have_scheduled_migration(5.minutes, 4) + expect(described_class::MIGRATION).to have_scheduled_migration(5.minutes, 1) + expect(described_class::MIGRATION).to have_scheduled_migration(5.minutes, 2) + expect(described_class::MIGRATION).to have_scheduled_migration(10.minutes, 3) + expect(described_class::MIGRATION).to have_scheduled_migration(10.minutes, 4) + end end end -- cgit v1.2.1 From 0c14b6f0985415bd9762886378b0ad5a009d80cd Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 12:14:41 +0200 Subject: Remove unused background migrations matcher --- .../migrate_stage_id_reference_in_background_spec.rb | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb index d3645ec0395..3eeca2e9659 100644 --- a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb +++ b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb @@ -2,18 +2,6 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20170628080858_migrate_stage_id_reference_in_background') describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do - matcher :have_migrated do |*expected| - match do |migration| - BackgroundMigrationWorker.jobs.any? do |job| - job['enqueued_at'].present? && job['args'] == [migration, expected] - end - end - - failure_message do |migration| - "Migration `#{migration}` with args `#{expected.inspect}` not executed!" - end - end - matcher :have_scheduled_migration do |delay, *expected| match do |migration| BackgroundMigrationWorker.jobs.any? do |job| -- cgit v1.2.1 From 8bd9cb6c87a1a1c51830360fe1aa1a228f9c768e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 12:14:54 +0200 Subject: Perform stage_id ref backgound migration in bulks --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 9e95216b35a..c54e8bde095 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -15,9 +15,9 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration Build.where(stage_id: nil) .in_batches(of: BATCH_SIZE) do |relation, index| schedule = index * 5.minutes - relation.each do |build| - BackgroundMigrationWorker.perform_at(schedule, MIGRATION, [build.id]) - end + jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } + + BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) end end -- cgit v1.2.1 From 42556419c99522e921299a3e247f115f580be7f1 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 12:26:37 +0200 Subject: Improve specs for background stage_id ref migration --- ...080858_migrate_stage_id_reference_in_background.rb | 11 +++++------ .../migrate_stage_id_reference_in_background_spec.rb | 19 +++++++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index c54e8bde095..1d95fc62c87 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -12,13 +12,12 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration end def up - Build.where(stage_id: nil) - .in_batches(of: BATCH_SIZE) do |relation, index| - schedule = index * 5.minutes - jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } + Build.where(stage_id: nil).in_batches(of: BATCH_SIZE) do |relation, index| + schedule = index * 5.minutes + jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } - BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) - end + BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) + end end def down diff --git a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb index 3eeca2e9659..046d9b351a8 100644 --- a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb +++ b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20170628080858_migrate_stage_id_reference_in_background') describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do - matcher :have_scheduled_migration do |delay, *expected| + matcher :be_scheduled_migration do |delay, *expected| match do |migration| BackgroundMigrationWorker.jobs.any? do |job| job['args'] == [migration, expected] && @@ -24,16 +24,22 @@ describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do stub_const('MigrateStageIdReferenceInBackground::BATCH_SIZE', 2) projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 345, name: 'gitlab2', path: 'gitlab2') + pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') + pipelines.create!(id: 2, project_id: 345, ref: 'feature', sha: 'cdf43c3c') jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test') jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy') + jobs.create!(id: 5, commit_id: 2, project_id: 345, stage_idx: 1, stage: 'test') stages.create(id: 101, pipeline_id: 1, project_id: 123, name: 'test') stages.create(id: 102, pipeline_id: 1, project_id: 123, name: 'build') stages.create(id: 103, pipeline_id: 1, project_id: 123, name: 'deploy') + + jobs.create!(id: 6, commit_id: 2, project_id: 345, stage_id: 101, stage_idx: 1, stage: 'test') end it 'correctly schedules background migrations' do @@ -41,10 +47,11 @@ describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do Timecop.freeze do migrate! - expect(described_class::MIGRATION).to have_scheduled_migration(5.minutes, 1) - expect(described_class::MIGRATION).to have_scheduled_migration(5.minutes, 2) - expect(described_class::MIGRATION).to have_scheduled_migration(10.minutes, 3) - expect(described_class::MIGRATION).to have_scheduled_migration(10.minutes, 4) + expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 1) + expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 2) + expect(described_class::MIGRATION).to be_scheduled_migration(10.minutes, 3) + expect(described_class::MIGRATION).to be_scheduled_migration(10.minutes, 4) + expect(BackgroundMigrationWorker.jobs.size).to eq 5 end end end @@ -55,7 +62,7 @@ describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do migrate! - expect(jobs.where(stage_id: nil)).to be_empty + expect(jobs.where(stage_id: nil)).to be_one end end end -- cgit v1.2.1 From 4687e07c6ea31821350aeb2c0dcb540f773b5268 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 14:31:12 +0200 Subject: Make `inline` a default sidekiq testing processing again --- spec/support/sidekiq.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/support/sidekiq.rb b/spec/support/sidekiq.rb index f3819ed2353..5478fea4e64 100644 --- a/spec/support/sidekiq.rb +++ b/spec/support/sidekiq.rb @@ -1,4 +1,4 @@ -require 'sidekiq/testing' +require 'sidekiq/testing/inline' Sidekiq::Testing.server_middleware do |chain| chain.add Gitlab::SidekiqStatus::ServerMiddleware -- cgit v1.2.1 From efa3511e0495f73b06e96e60538f8e719ffe97a4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 15:20:26 +0200 Subject: Test if argument passed to a migration is present --- lib/gitlab/background_migration/migrate_build_stage_id_reference.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb index 87c6c4ed49f..711126ea0d3 100644 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -2,7 +2,7 @@ module Gitlab module BackgroundMigration class MigrateBuildStageIdReference def perform(id) - raise ArgumentError unless id.is_a?(Integer) + raise ArgumentError unless id.present? sql = <<-SQL.strip_heredoc UPDATE "ci_builds" SET "stage_id" = ( -- cgit v1.2.1 From c7f6e5efb51d28242bfe5102b9db9cd7a6c1e24b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 15:22:06 +0200 Subject: Use integers to schedule delayed background migrations --- app/workers/background_migration_worker.rb | 4 ++-- spec/migrations/migrate_stage_id_reference_in_background_spec.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index 23c297de8bc..e6ca1159b38 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -17,8 +17,8 @@ class BackgroundMigrationWorker # Schedules multiple jobs in bulk, with a delay. # def self.perform_bulk_in(delay, jobs) - now = Time.now.to_f - schedule = now + delay.to_f + now = Time.now.to_i + schedule = now + delay.to_i raise ArgumentError if schedule <= now diff --git a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb index 046d9b351a8..1bd2c14b61c 100644 --- a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb +++ b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb @@ -6,7 +6,7 @@ describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do match do |migration| BackgroundMigrationWorker.jobs.any? do |job| job['args'] == [migration, expected] && - job['at'].to_f == (delay.to_f + Time.now.to_f) + job['at'].to_f == (delay.to_i + Time.now.to_i) end end -- cgit v1.2.1 From 6db8253cb86f44a80282706cc3de3df954661434 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 15:26:47 +0200 Subject: Improve readability of build stage id migration query --- .../migrate_build_stage_id_reference.rb | 13 +++++++------ .../migrate_stage_id_reference_in_background_spec.rb | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb index 711126ea0d3..c8669ca3272 100644 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -5,12 +5,13 @@ module Gitlab raise ArgumentError unless id.present? sql = <<-SQL.strip_heredoc - UPDATE "ci_builds" SET "stage_id" = ( - SELECT id FROM ci_stages - WHERE ci_stages.pipeline_id = ci_builds.commit_id - AND ci_stages.name = ci_builds.stage - ) - WHERE "ci_builds"."id" = #{id} AND "ci_builds"."stage_id" IS NULL + UPDATE "ci_builds" + SET "stage_id" = + (SELECT id FROM ci_stages + WHERE ci_stages.pipeline_id = ci_builds.commit_id + AND ci_stages.name = ci_builds.stage) + WHERE "ci_builds"."id" = #{id} + AND "ci_builds"."stage_id" IS NULL SQL ActiveRecord::Base.connection.execute(sql) diff --git a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb index 1bd2c14b61c..63787d71233 100644 --- a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb +++ b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb @@ -58,11 +58,11 @@ describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do it 'schedules background migrations' do Sidekiq::Testing.inline! do - expect(jobs.where(stage_id: nil)).to be_present + expect(jobs.where(stage_id: nil).count).to eq 5 migrate! - expect(jobs.where(stage_id: nil)).to be_one + expect(jobs.where(stage_id: nil).count).to eq 1 end end end -- cgit v1.2.1 From 9c2290315e354c854bcd2ef0fed784dc5cfb48e8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 15:36:39 +0200 Subject: Do not compare float with integer in migrations specs --- spec/migrations/migrate_stage_id_reference_in_background_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb index 63787d71233..2e5504c849d 100644 --- a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb +++ b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb @@ -6,7 +6,7 @@ describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do match do |migration| BackgroundMigrationWorker.jobs.any? do |job| job['args'] == [migration, expected] && - job['at'].to_f == (delay.to_i + Time.now.to_i) + job['at'].to_i == (delay.to_i + Time.now.to_i) end end -- cgit v1.2.1 From 1edd063a6c338e889d95def3e31aaca5420bfa5a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 30 Jun 2017 12:05:33 +0200 Subject: Add description to exception in bg migrations worker --- app/workers/background_migration_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index e6ca1159b38..e7ed71a687c 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -20,7 +20,7 @@ class BackgroundMigrationWorker now = Time.now.to_i schedule = now + delay.to_i - raise ArgumentError if schedule <= now + raise ArgumentError, 'Delay time invalid!' if schedule <= now Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], -- cgit v1.2.1 From 1bb0448f86c106a70a82627b3ffe84ed06a59081 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 30 Jun 2017 12:06:29 +0200 Subject: Improve code examples in background migrations docs --- doc/development/background_migrations.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/development/background_migrations.md b/doc/development/background_migrations.md index a58f161fc30..72a34aa7de9 100644 --- a/doc/development/background_migrations.md +++ b/doc/development/background_migrations.md @@ -56,8 +56,7 @@ Usually it's better to enqueue jobs in bulk, for this you can use ```ruby BackgroundMigrationWorker.perform_bulk( [['BackgroundMigrationClassName', [1]], - ['BackgroundMigrationClassName', [2]], - ...] + ['BackgroundMigrationClassName', [2]]] ) ``` @@ -73,8 +72,7 @@ If you would like to schedule jobs in bulk with a delay, you can use ```ruby jobs = [['BackgroundMigrationClassName', [1]], - ['BackgroundMigrationClassName', [2]], - ...] + ['BackgroundMigrationClassName', [2]]] BackgroundMigrationWorker.perform_bulk_in(5.minutes, jobs) ``` -- cgit v1.2.1 From 6997dfa3426b26f7eb8f294b261827ef0b6d823b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 30 Jun 2017 12:07:31 +0200 Subject: Sanitize id value passed to async background migration --- lib/gitlab/background_migration/migrate_build_stage_id_reference.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb index c8669ca3272..d1d0a968588 100644 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -2,15 +2,13 @@ module Gitlab module BackgroundMigration class MigrateBuildStageIdReference def perform(id) - raise ArgumentError unless id.present? - sql = <<-SQL.strip_heredoc UPDATE "ci_builds" SET "stage_id" = (SELECT id FROM ci_stages WHERE ci_stages.pipeline_id = ci_builds.commit_id AND ci_stages.name = ci_builds.stage) - WHERE "ci_builds"."id" = #{id} + WHERE "ci_builds"."id" = #{id.to_i} AND "ci_builds"."stage_id" IS NULL SQL -- cgit v1.2.1 From 134f204ed8f5dd80e44463338ae93f3d905ca7af Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 30 Jun 2017 13:03:47 +0200 Subject: Do not override original AR5 batching interface --- config/initializers/ar5_batching.rb | 4 ++-- .../20170628080858_migrate_stage_id_reference_in_background.rb | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/config/initializers/ar5_batching.rb b/config/initializers/ar5_batching.rb index 31efef83a6f..35e8b3808e2 100644 --- a/config/initializers/ar5_batching.rb +++ b/config/initializers/ar5_batching.rb @@ -15,7 +15,7 @@ module ActiveRecord relation = relation.where(arel_table[primary_key].lteq(finish)) if finish batch_relation = relation - 1.step do |index| + loop do if load records = batch_relation.records ids = records.map(&:id) @@ -31,7 +31,7 @@ module ActiveRecord primary_key_offset = ids.last raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset - yield yielded_relation, index + yield yielded_relation break if ids.length < of batch_relation = relation.where(arel_table[primary_key].gt(primary_key_offset)) diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 1d95fc62c87..30849ea1361 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -12,9 +12,12 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration end def up - Build.where(stage_id: nil).in_batches(of: BATCH_SIZE) do |relation, index| - schedule = index * 5.minutes + index = 1 + + Build.where(stage_id: nil).in_batches(of: BATCH_SIZE) do |relation| jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } + schedule = index * 5.minutes + index += 1 BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) end -- cgit v1.2.1 From 9c7c95c768cd5294dd085c2fc2425fae91c4c689 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 30 Jun 2017 14:22:23 +0200 Subject: Add initial changes for stages statuses migration --- .../20170630105320_add_status_to_ci_stages.rb | 9 +++ .../20170630111158_migrate_stages_statuses.rb | 76 ++++++++++++++++++++++ db/schema.rb | 3 +- spec/migrations/migrate_stages_statuses_spec.rb | 50 ++++++++++++++ 4 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20170630105320_add_status_to_ci_stages.rb create mode 100644 db/post_migrate/20170630111158_migrate_stages_statuses.rb create mode 100644 spec/migrations/migrate_stages_statuses_spec.rb diff --git a/db/migrate/20170630105320_add_status_to_ci_stages.rb b/db/migrate/20170630105320_add_status_to_ci_stages.rb new file mode 100644 index 00000000000..d497a61a959 --- /dev/null +++ b/db/migrate/20170630105320_add_status_to_ci_stages.rb @@ -0,0 +1,9 @@ +class AddStatusToCiStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :ci_stages, :status, :integer + end +end diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb new file mode 100644 index 00000000000..b4b76893595 --- /dev/null +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -0,0 +1,76 @@ +class MigrateStagesStatuses < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + + scope :relevant, -> do + where(status: %w[pending running success failed canceled skipped manual]) + end + + scope :created, -> { where(status: 'created') } + scope :running, -> { where(status: 'running') } + scope :pending, -> { where(status: 'pending') } + scope :success, -> { where(status: 'success') } + scope :failed, -> { where(status: 'failed') } + scope :canceled, -> { where(status: 'canceled') } + scope :skipped, -> { where(status: 'skipped') } + scope :manual, -> { where(status: 'manual') } + + scope :failed_but_allowed, -> do + where(allow_failure: true, status: [:failed, :canceled]) + end + + scope :exclude_ignored, -> do + where("allow_failure = ? OR status IN (?)", + false, all_state_names - [:failed, :canceled, :manual]) + end + + def status_sql + scope_relevant = relevant.exclude_ignored + scope_warnings = relevant.failed_but_allowed + + builds = scope_relevant.select('count(*)').to_sql + created = scope_relevant.created.select('count(*)').to_sql + success = scope_relevant.success.select('count(*)').to_sql + manual = scope_relevant.manual.select('count(*)').to_sql + pending = scope_relevant.pending.select('count(*)').to_sql + running = scope_relevant.running.select('count(*)').to_sql + skipped = scope_relevant.skipped.select('count(*)').to_sql + canceled = scope_relevant.canceled.select('count(*)').to_sql + warnings = scope_warnings.select('count(*) > 0').to_sql + + "(CASE + WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN 'success' + WHEN (#{builds})=(#{skipped}) THEN 'skipped' + WHEN (#{builds})=(#{success}) THEN 'success' + WHEN (#{builds})=(#{created}) THEN 'created' + WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success' + WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' + WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' + WHEN (#{running})+(#{pending})>0 THEN 'running' + WHEN (#{manual})>0 THEN 'manual' + WHEN (#{created})>0 THEN 'running' + ELSE 'failed' + END)" + end + end + + def up + execute <<-SQL.strip_heredoc + SQL + end + + def down + execute <<-SQL.strip_heredoc + UPDATE ci_stages SET status = null + SQL + end + + private + +end diff --git a/db/schema.rb b/db/schema.rb index 8c7440ee610..f34dd32fb74 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170622162730) do +ActiveRecord::Schema.define(version: 20170630111158) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -337,6 +337,7 @@ ActiveRecord::Schema.define(version: 20170622162730) do t.datetime "created_at" t.datetime "updated_at" t.string "name" + t.integer "status" end add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree diff --git a/spec/migrations/migrate_stages_statuses_spec.rb b/spec/migrations/migrate_stages_statuses_spec.rb new file mode 100644 index 00000000000..dc54f4acbf4 --- /dev/null +++ b/spec/migrations/migrate_stages_statuses_spec.rb @@ -0,0 +1,50 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20170630111158_migrate_stages_statuses.rb') + +describe MigrateStagesStatuses, :migration do + let(:jobs) { table(:ci_builds) } + let(:stages) { table(:ci_stages) } + let(:pipelines) { table(:ci_pipelines) } + let(:projects) { table(:projects) } + + STATUSES = { created: 0, pending: 1, running: 2, success: 3, + failed: 4, canceled: 5, skipped: 6, manual: 7 } + STAGES = { test: 1, build: 2, deploy: 3} + + before do + projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 2, name: 'gitlab2', path: 'gitlab2') + + pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') + pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb') + + create_job(project: 1, pipeline: 1, stage: 'test', status: 'success') + create_job(project: 1, pipeline: 1, stage: 'test', status: 'running') + create_job(project: 1, pipeline: 1, stage: 'build', status: 'success') + create_job(project: 1, pipeline: 1, stage: 'build', status: 'failed') + create_job(project: 2, pipeline: 2, stage: 'test', status: 'success') + create_job(project: 2, pipeline: 2, stage: 'test', status: 'succcss') + + stages.create!(id: 1, pipeline_id: 1, project_id: 1, status: nil) + stages.create!(id: 2, pipeline_id: 1, project_id: 1, status: nil) + stages.create!(id: 3, pipeline_id: 2, project_id: 2, status: nil) + end + + pending 'correctly migrates stages statuses' do + expect(stages.where(status: nil).count).to eq 3 + + migrate! + + expect(stages.where(status: nil)).to be_empty + expect(stages.all.order(:id, :asc).pluck(:stage)) + .to eq %w[running success failed] + end + + def create_job(project:, pipeline:, stage:, status:) + stage_idx = STAGES[stage.to_sym] + status_id = STATUSES[status.to_sym] + + jobs.create!(project_id: project, commit_id: pipeline, + stage_idx: stage_idx, stage: stage, status: status_id) + end +end -- cgit v1.2.1 From a078767223ca9e66a9d5dbf614b2efc8bf7c45d4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Jul 2017 13:00:38 +0200 Subject: Improve exception description in bg migrations --- app/workers/background_migration_worker.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index e7ed71a687c..45ce49bb5c0 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -20,7 +20,9 @@ class BackgroundMigrationWorker now = Time.now.to_i schedule = now + delay.to_i - raise ArgumentError, 'Delay time invalid!' if schedule <= now + if schedule <= now + raise ArgumentError, 'The schedule time must be in the future!' + end Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], -- cgit v1.2.1 From f6966d96ec5941db364a2c8d9d2d80d3aa7d20f2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Jul 2017 13:02:51 +0200 Subject: Reduce a delay between stage_id scheduled migrations --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 2 +- spec/migrations/migrate_stage_id_reference_in_background_spec.rb | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 30849ea1361..ebec4cb6bb7 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -16,7 +16,7 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration Build.where(stage_id: nil).in_batches(of: BATCH_SIZE) do |relation| jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } - schedule = index * 5.minutes + schedule = index * 2.minutes index += 1 BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) diff --git a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb index 2e5504c849d..a32a7fceb68 100644 --- a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb +++ b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb @@ -47,10 +47,10 @@ describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do Timecop.freeze do migrate! - expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 1) - expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 2) - expect(described_class::MIGRATION).to be_scheduled_migration(10.minutes, 3) - expect(described_class::MIGRATION).to be_scheduled_migration(10.minutes, 4) + expect(described_class::MIGRATION).to be_scheduled_migration(2.minutes, 1) + expect(described_class::MIGRATION).to be_scheduled_migration(2.minutes, 2) + expect(described_class::MIGRATION).to be_scheduled_migration(4.minutes, 3) + expect(described_class::MIGRATION).to be_scheduled_migration(4.minutes, 4) expect(BackgroundMigrationWorker.jobs.size).to eq 5 end end -- cgit v1.2.1 From 7103c4a707157594c261ba2f68fbb649ca4df769 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Jul 2017 09:20:18 +0200 Subject: Extend stages statuses migration --- .../20170630111158_migrate_stages_statuses.rb | 55 +++++++++++++--------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index b4b76893595..8c6de84adf5 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -5,13 +5,17 @@ class MigrateStagesStatuses < ActiveRecord::Migration disable_ddl_transaction! + STATUSES = { created: 0, pending: 1, running: 2, success: 3, + failed: 4, canceled: 5, skipped: 6, manual: 7 } + + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' + end + class Build < ActiveRecord::Base self.table_name = 'ci_builds' - scope :relevant, -> do - where(status: %w[pending running success failed canceled skipped manual]) - end - + scope :latest, -> { where(retried: [false, nil]) } scope :created, -> { where(status: 'created') } scope :running, -> { where(status: 'running') } scope :pending, -> { where(status: 'pending') } @@ -27,12 +31,12 @@ class MigrateStagesStatuses < ActiveRecord::Migration scope :exclude_ignored, -> do where("allow_failure = ? OR status IN (?)", - false, all_state_names - [:failed, :canceled, :manual]) + false, %w[created pending running success skipped]) end - def status_sql - scope_relevant = relevant.exclude_ignored - scope_warnings = relevant.failed_but_allowed + def self.status_sql + scope_relevant = latest.exclude_ignored + scope_warnings = latest.failed_but_allowed builds = scope_relevant.select('count(*)').to_sql created = scope_relevant.created.select('count(*)').to_sql @@ -45,24 +49,33 @@ class MigrateStagesStatuses < ActiveRecord::Migration warnings = scope_warnings.select('count(*) > 0').to_sql "(CASE - WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN 'success' - WHEN (#{builds})=(#{skipped}) THEN 'skipped' - WHEN (#{builds})=(#{success}) THEN 'success' - WHEN (#{builds})=(#{created}) THEN 'created' - WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success' - WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' - WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' - WHEN (#{running})+(#{pending})>0 THEN 'running' - WHEN (#{manual})>0 THEN 'manual' - WHEN (#{created})>0 THEN 'running' - ELSE 'failed' + WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} + WHEN (#{builds})=(#{skipped}) THEN #{STATUSES[:skipped]} + WHEN (#{builds})=(#{success}) THEN #{STATUSES[:success]} + WHEN (#{builds})=(#{created}) THEN #{STATUSES[:created]} + WHEN (#{builds})=(#{success})+(#{skipped}) THEN #{STATUSES[:success]} + WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN #{STATUSES[:canceled]} + WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN #{STATUSES[:pending]} + WHEN (#{running})+(#{pending})>0 THEN '#{STATUSES[:running]} + WHEN (#{manual})>0 THEN #{STATUSES[:manual]} + WHEN (#{created})>0 THEN #{STATUSES[:running]} + ELSE #{STATUSES[:failed]} END)" end end def up - execute <<-SQL.strip_heredoc - SQL + Stage.all.in_batches(of: 10000) do |relation| + status_sql = Build + .where('ci_builds.commit_id = ci_stages.pipeline_id') + .where('ci_builds.stage = ci_stages.name') + .status_sql + + execute <<-SQL.strip_heredoc + UPDATE ci_stages SET status = #{status_sql} + WHERE id = (#{relation.select(:id).to_sql}) + SQL + end end def down -- cgit v1.2.1 From d60ce6e9f44eba769a6ad595014ae96095169dd2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Jul 2017 13:43:26 +0200 Subject: Implement initial working stages statuses migration --- .../20170630111158_migrate_stages_statuses.rb | 46 ++++++++++------------ spec/migrations/migrate_stages_statuses_spec.rb | 21 +++++----- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index 8c6de84adf5..62542ed0001 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -48,34 +48,31 @@ class MigrateStagesStatuses < ActiveRecord::Migration canceled = scope_relevant.canceled.select('count(*)').to_sql warnings = scope_warnings.select('count(*) > 0').to_sql - "(CASE - WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} - WHEN (#{builds})=(#{skipped}) THEN #{STATUSES[:skipped]} - WHEN (#{builds})=(#{success}) THEN #{STATUSES[:success]} - WHEN (#{builds})=(#{created}) THEN #{STATUSES[:created]} - WHEN (#{builds})=(#{success})+(#{skipped}) THEN #{STATUSES[:success]} - WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN #{STATUSES[:canceled]} - WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN #{STATUSES[:pending]} - WHEN (#{running})+(#{pending})>0 THEN '#{STATUSES[:running]} - WHEN (#{manual})>0 THEN #{STATUSES[:manual]} - WHEN (#{created})>0 THEN #{STATUSES[:running]} - ELSE #{STATUSES[:failed]} - END)" + <<-SQL.strip_heredoc + (CASE + WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]} + WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]} + WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]} + WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]} + WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]} + WHEN (#{created}) > 0 THEN #{STATUSES[:running]} + ELSE #{STATUSES[:failed]} + END) + SQL end end def up - Stage.all.in_batches(of: 10000) do |relation| - status_sql = Build - .where('ci_builds.commit_id = ci_stages.pipeline_id') - .where('ci_builds.stage = ci_stages.name') - .status_sql + status_sql = Build + .where('ci_builds.commit_id = ci_stages.pipeline_id') + .where('ci_builds.stage = ci_stages.name') + .status_sql - execute <<-SQL.strip_heredoc - UPDATE ci_stages SET status = #{status_sql} - WHERE id = (#{relation.select(:id).to_sql}) - SQL - end + update_column_in_batches(:ci_stages, :status, Arel.sql("(#{status_sql})")) end def down @@ -83,7 +80,4 @@ class MigrateStagesStatuses < ActiveRecord::Migration UPDATE ci_stages SET status = null SQL end - - private - end diff --git a/spec/migrations/migrate_stages_statuses_spec.rb b/spec/migrations/migrate_stages_statuses_spec.rb index dc54f4acbf4..95fa2977b31 100644 --- a/spec/migrations/migrate_stages_statuses_spec.rb +++ b/spec/migrations/migrate_stages_statuses_spec.rb @@ -15,36 +15,35 @@ describe MigrateStagesStatuses, :migration do projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1') projects.create!(id: 2, name: 'gitlab2', path: 'gitlab2') - pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') - pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb') + pipelines.create!(id: 1, project_id: 1, ref: 'master', sha: 'adf43c3a') + pipelines.create!(id: 2, project_id: 2, ref: 'feature', sha: '21a3deb') create_job(project: 1, pipeline: 1, stage: 'test', status: 'success') create_job(project: 1, pipeline: 1, stage: 'test', status: 'running') create_job(project: 1, pipeline: 1, stage: 'build', status: 'success') create_job(project: 1, pipeline: 1, stage: 'build', status: 'failed') create_job(project: 2, pipeline: 2, stage: 'test', status: 'success') - create_job(project: 2, pipeline: 2, stage: 'test', status: 'succcss') + create_job(project: 2, pipeline: 2, stage: 'test', status: 'success') - stages.create!(id: 1, pipeline_id: 1, project_id: 1, status: nil) - stages.create!(id: 2, pipeline_id: 1, project_id: 1, status: nil) - stages.create!(id: 3, pipeline_id: 2, project_id: 2, status: nil) + stages.create!(id: 1, pipeline_id: 1, project_id: 1, name: 'test', status: nil) + stages.create!(id: 2, pipeline_id: 1, project_id: 1, name: 'build', status: nil) + stages.create!(id: 3, pipeline_id: 2, project_id: 2, name: 'test', status: nil) end - pending 'correctly migrates stages statuses' do + it 'correctly migrates stages statuses' do expect(stages.where(status: nil).count).to eq 3 migrate! expect(stages.where(status: nil)).to be_empty - expect(stages.all.order(:id, :asc).pluck(:stage)) - .to eq %w[running success failed] + expect(stages.all.order('id ASC').pluck(:status)) + .to eq [STATUSES[:running], STATUSES[:failed], STATUSES[:success]] end def create_job(project:, pipeline:, stage:, status:) stage_idx = STAGES[stage.to_sym] - status_id = STATUSES[status.to_sym] jobs.create!(project_id: project, commit_id: pipeline, - stage_idx: stage_idx, stage: stage, status: status_id) + stage_idx: stage_idx, stage: stage, status: status) end end -- cgit v1.2.1 From f9228f6bf46f1d1caa4c62b80b8bd6ec883d33ae Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Jul 2017 14:07:17 +0200 Subject: Add a test for stage status migration with retried jobs --- spec/migrations/migrate_stages_statuses_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/migrations/migrate_stages_statuses_spec.rb b/spec/migrations/migrate_stages_statuses_spec.rb index 95fa2977b31..81bc38fea10 100644 --- a/spec/migrations/migrate_stages_statuses_spec.rb +++ b/spec/migrations/migrate_stages_statuses_spec.rb @@ -24,6 +24,7 @@ describe MigrateStagesStatuses, :migration do create_job(project: 1, pipeline: 1, stage: 'build', status: 'failed') create_job(project: 2, pipeline: 2, stage: 'test', status: 'success') create_job(project: 2, pipeline: 2, stage: 'test', status: 'success') + create_job(project: 2, pipeline: 2, stage: 'test', status: 'failed', retried: true) stages.create!(id: 1, pipeline_id: 1, project_id: 1, name: 'test', status: nil) stages.create!(id: 2, pipeline_id: 1, project_id: 1, name: 'build', status: nil) @@ -40,10 +41,9 @@ describe MigrateStagesStatuses, :migration do .to eq [STATUSES[:running], STATUSES[:failed], STATUSES[:success]] end - def create_job(project:, pipeline:, stage:, status:) - stage_idx = STAGES[stage.to_sym] - + def create_job(project:, pipeline:, stage:, status:, **opts) jobs.create!(project_id: project, commit_id: pipeline, - stage_idx: stage_idx, stage: stage, status: status) + stage_idx: STAGES[stage.to_sym], stage: stage, + status: status, **opts) end end -- cgit v1.2.1 From a17c90b2a7331a7427813684b04095b55c4b3cc1 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Jul 2017 15:31:15 +0200 Subject: Use enumerated status in persisted stage class --- app/models/ci/stage.rb | 3 +++ app/models/concerns/has_status.rb | 10 ++++++++++ spec/factories/ci/stages.rb | 6 ++++++ spec/migrations/migrate_stages_statuses_spec.rb | 5 +++-- spec/models/ci/stage_spec.rb | 21 +++++++++++++++++++++ 5 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 spec/models/ci/stage_spec.rb diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 59570924c8d..0c7f8c7f485 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -1,6 +1,9 @@ module Ci class Stage < ActiveRecord::Base extend Ci::Model + include HasStatus + + enumerated_status! belongs_to :project belongs_to :pipeline diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 32af5566135..235196cae13 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -8,6 +8,8 @@ module HasStatus ACTIVE_STATUSES = %w[pending running].freeze COMPLETED_STATUSES = %w[success failed canceled skipped].freeze ORDERED_STATUSES = %w[failed pending running manual canceled success skipped created].freeze + STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, + failed: 4, canceled: 5, skipped: 6, manual: 7 } class_methods do def status_sql @@ -54,6 +56,14 @@ module HasStatus def all_state_names state_machines.values.flat_map(&:states).flat_map { |s| s.map(&:name) } end + + private + + def enumerated_status! + @status_strategy = :enumerator + + enum status: HasStatus::STATUSES_ENUM + end end included do diff --git a/spec/factories/ci/stages.rb b/spec/factories/ci/stages.rb index d3c8bf9d54f..ee8ac85c92e 100644 --- a/spec/factories/ci/stages.rb +++ b/spec/factories/ci/stages.rb @@ -15,4 +15,10 @@ FactoryGirl.define do warnings: warnings) end end + + factory :ci_stage_entity, class: Ci::Stage do + project factory: :empty_project + pipeline factory: :ci_empty_pipeline + status 'pending' + end end diff --git a/spec/migrations/migrate_stages_statuses_spec.rb b/spec/migrations/migrate_stages_statuses_spec.rb index 81bc38fea10..478ddad262a 100644 --- a/spec/migrations/migrate_stages_statuses_spec.rb +++ b/spec/migrations/migrate_stages_statuses_spec.rb @@ -9,7 +9,6 @@ describe MigrateStagesStatuses, :migration do STATUSES = { created: 0, pending: 1, running: 2, success: 3, failed: 4, canceled: 5, skipped: 6, manual: 7 } - STAGES = { test: 1, build: 2, deploy: 3} before do projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1') @@ -42,8 +41,10 @@ describe MigrateStagesStatuses, :migration do end def create_job(project:, pipeline:, stage:, status:, **opts) + stages = { test: 1, build: 2, deploy: 3} + jobs.create!(project_id: project, commit_id: pipeline, - stage_idx: STAGES[stage.to_sym], stage: stage, + stage_idx: stages[stage.to_sym], stage: stage, status: status, **opts) end end diff --git a/spec/models/ci/stage_spec.rb b/spec/models/ci/stage_spec.rb new file mode 100644 index 00000000000..911c468ff1a --- /dev/null +++ b/spec/models/ci/stage_spec.rb @@ -0,0 +1,21 @@ +require 'spec_helper' + +describe Ci::Stage, :models do + describe '#status' do + context 'when stage is pending' do + let(:stage) { create(:ci_stage_entity, status: 'pending') } + + it 'has a correct status value' do + expect(stage.status).to eq 'pending' + end + end + + context 'when stage is success' do + let(:stage) { create(:ci_stage_entity, status: 'success') } + + it 'has a correct status value' do + expect(stage.status).to eq 'success' + end + end + end +end -- cgit v1.2.1 From 22d8460b5d9926d7608d23aeb58e20d9035efa92 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Jul 2017 15:50:03 +0200 Subject: Add some validations to persisted stage model --- app/models/ci/stage.rb | 5 +++++ app/models/concerns/has_status.rb | 12 ++++-------- spec/factories/ci/stages.rb | 2 ++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 0c7f8c7f485..da1c3753924 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -1,6 +1,7 @@ module Ci class Stage < ActiveRecord::Base extend Ci::Model + include Importable include HasStatus enumerated_status! @@ -10,5 +11,9 @@ module Ci has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id has_many :builds, foreign_key: :commit_id + + validates :project, presence: true, unless: :importing? + validates :pipeline, presence: true, unless: :importing? + validates :name, presence: true, unless: :importing? end end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 235196cae13..8ea5a007f76 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -12,6 +12,10 @@ module HasStatus failed: 4, canceled: 5, skipped: 6, manual: 7 } class_methods do + def enumerated_status! + enum status: HasStatus::STATUSES_ENUM + end + def status_sql scope_relevant = respond_to?(:exclude_ignored) ? exclude_ignored : all scope_warnings = respond_to?(:failed_but_allowed) ? failed_but_allowed : none @@ -56,14 +60,6 @@ module HasStatus def all_state_names state_machines.values.flat_map(&:states).flat_map { |s| s.map(&:name) } end - - private - - def enumerated_status! - @status_strategy = :enumerator - - enum status: HasStatus::STATUSES_ENUM - end end included do diff --git a/spec/factories/ci/stages.rb b/spec/factories/ci/stages.rb index ee8ac85c92e..bace932cf99 100644 --- a/spec/factories/ci/stages.rb +++ b/spec/factories/ci/stages.rb @@ -19,6 +19,8 @@ FactoryGirl.define do factory :ci_stage_entity, class: Ci::Stage do project factory: :empty_project pipeline factory: :ci_empty_pipeline + + name 'test' status 'pending' end end -- cgit v1.2.1 From b3ee172b4ee6fd22ebf3705edf6762a9dd777cdc Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Jul 2017 16:41:33 +0200 Subject: Add a workaround for a tmp job -> stage relation We still didn't migrate `ci_builds.stage_id`, so we can't use a belongs_to association. We also have `ci_builds.stage` string attribute, that we need to phase out in favor of `ci_stages.name`. --- app/models/commit_status.rb | 9 +++++++++ spec/models/commit_status_spec.rb | 18 +++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 07cec63b939..afdc75f75fb 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -38,6 +38,15 @@ class CommitStatus < ActiveRecord::Base scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) } scope :after_stage, -> (index) { where('stage_idx > ?', index) } + ## + # TODO, we will change this to `belongs_to :stage` when we phase out + # `ci_builds.stage` attribute and migrate `ci_builds.stage_id` reference in + # one of upcoming releases. + # + def stage_entity + Ci::Stage.find_by(pipeline: pipeline, name: stage) + end + state_machine :status do event :enqueue do transition [:created, :skipped, :manual] => :pending diff --git a/spec/models/commit_status_spec.rb b/spec/models/commit_status_spec.rb index 1e074c7ad26..c0cbf0b2f95 100644 --- a/spec/models/commit_status_spec.rb +++ b/spec/models/commit_status_spec.rb @@ -7,10 +7,10 @@ describe CommitStatus, :models do create(:ci_pipeline, project: project, sha: project.commit.id) end - let(:commit_status) { create_status } + let(:commit_status) { create_status(stage: 'test') } - def create_status(args = {}) - create(:commit_status, args.merge(pipeline: pipeline)) + def create_status(**opts) + create(:commit_status, pipeline: pipeline, **opts) end it { is_expected.to belong_to(:pipeline) } @@ -408,6 +408,18 @@ describe CommitStatus, :models do end end + describe '#stage_entity' do + let!(:stage) do + create(:ci_stage_entity, pipeline: commit_status.pipeline, + name: commit_status.stage) + end + + it 'has a correct association with persisted stage' do + expect(commit_status.stage_entity).to eq stage + end + end + + describe '#locking_enabled?' do before do commit_status.lock_version = 100 -- cgit v1.2.1 From 93d217bda639b94c129afd71343e429f935a4ada Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Jul 2017 16:58:35 +0200 Subject: Migrate only old stages without status that is set --- db/post_migrate/20170630111158_migrate_stages_statuses.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index 62542ed0001..9dac91960ff 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -72,7 +72,9 @@ class MigrateStagesStatuses < ActiveRecord::Migration .where('ci_builds.stage = ci_stages.name') .status_sql - update_column_in_batches(:ci_stages, :status, Arel.sql("(#{status_sql})")) + update_column_in_batches(:ci_stages, :status, Arel.sql("(#{status_sql})")) do |table, query| + query.where(table[:status].eq(nil)) + end end def down -- cgit v1.2.1 From c5f1e1a70bd79b36fe8cfda75b7366dd8ee90d66 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Jul 2017 09:11:15 +0200 Subject: Disable statement timeout in stages statuses migration --- db/post_migrate/20170630111158_migrate_stages_statuses.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index 9dac91960ff..c0a5294720d 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -8,10 +8,6 @@ class MigrateStagesStatuses < ActiveRecord::Migration STATUSES = { created: 0, pending: 1, running: 2, success: 3, failed: 4, canceled: 5, skipped: 6, manual: 7 } - class Stage < ActiveRecord::Base - self.table_name = 'ci_stages' - end - class Build < ActiveRecord::Base self.table_name = 'ci_builds' @@ -67,6 +63,8 @@ class MigrateStagesStatuses < ActiveRecord::Migration end def up + disable_statement_timeout + status_sql = Build .where('ci_builds.commit_id = ci_stages.pipeline_id') .where('ci_builds.stage = ci_stages.name') @@ -78,6 +76,8 @@ class MigrateStagesStatuses < ActiveRecord::Migration end def down + disable_statement_timeout + execute <<-SQL.strip_heredoc UPDATE ci_stages SET status = null SQL -- cgit v1.2.1 From 6c477d5b9496829eb5cb56ef32a0dd813be7dc16 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Jul 2017 10:54:48 +0200 Subject: Move stages status migration to the background worker --- .../20170630111158_migrate_stages_statuses.rb | 71 +++----------------- .../background_migration/migrate_stage_status.rb | 76 ++++++++++++++++++++++ ...igrate_stage_id_reference_in_background_spec.rb | 13 ---- spec/migrations/migrate_stages_statuses_spec.rb | 28 ++++++-- spec/support/background_migrations_matchers.rb | 13 ++++ 5 files changed, 122 insertions(+), 79 deletions(-) create mode 100644 lib/gitlab/background_migration/migrate_stage_status.rb create mode 100644 spec/support/background_migrations_matchers.rb diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index c0a5294720d..2bc067e5d90 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -5,73 +5,22 @@ class MigrateStagesStatuses < ActiveRecord::Migration disable_ddl_transaction! - STATUSES = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7 } + BATCH_SIZE = 10000 + MIGRATION = 'MigrateStageStatus'.freeze - class Build < ActiveRecord::Base - self.table_name = 'ci_builds' - - scope :latest, -> { where(retried: [false, nil]) } - scope :created, -> { where(status: 'created') } - scope :running, -> { where(status: 'running') } - scope :pending, -> { where(status: 'pending') } - scope :success, -> { where(status: 'success') } - scope :failed, -> { where(status: 'failed') } - scope :canceled, -> { where(status: 'canceled') } - scope :skipped, -> { where(status: 'skipped') } - scope :manual, -> { where(status: 'manual') } - - scope :failed_but_allowed, -> do - where(allow_failure: true, status: [:failed, :canceled]) - end - - scope :exclude_ignored, -> do - where("allow_failure = ? OR status IN (?)", - false, %w[created pending running success skipped]) - end - - def self.status_sql - scope_relevant = latest.exclude_ignored - scope_warnings = latest.failed_but_allowed - - builds = scope_relevant.select('count(*)').to_sql - created = scope_relevant.created.select('count(*)').to_sql - success = scope_relevant.success.select('count(*)').to_sql - manual = scope_relevant.manual.select('count(*)').to_sql - pending = scope_relevant.pending.select('count(*)').to_sql - running = scope_relevant.running.select('count(*)').to_sql - skipped = scope_relevant.skipped.select('count(*)').to_sql - canceled = scope_relevant.canceled.select('count(*)').to_sql - warnings = scope_warnings.select('count(*) > 0').to_sql - - <<-SQL.strip_heredoc - (CASE - WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]} - WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]} - WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]} - WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]} - WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]} - WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]} - WHEN (#{created}) > 0 THEN #{STATUSES[:running]} - ELSE #{STATUSES[:failed]} - END) - SQL - end + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' end def up - disable_statement_timeout + index = 1 - status_sql = Build - .where('ci_builds.commit_id = ci_stages.pipeline_id') - .where('ci_builds.stage = ci_stages.name') - .status_sql + Stage.where(status: nil).in_batches(of: BATCH_SIZE) do |relation| + jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } + schedule = index * 5.minutes + index += 1 - update_column_in_batches(:ci_stages, :status, Arel.sql("(#{status_sql})")) do |table, query| - query.where(table[:status].eq(nil)) + BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) end end diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb new file mode 100644 index 00000000000..e4fdc723b13 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_stage_status.rb @@ -0,0 +1,76 @@ +module Gitlab + module BackgroundMigration + class MigrateStageStatus + STATUSES = { created: 0, pending: 1, running: 2, success: 3, + failed: 4, canceled: 5, skipped: 6, manual: 7 } + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + + scope :latest, -> { where(retried: [false, nil]) } + scope :created, -> { where(status: 'created') } + scope :running, -> { where(status: 'running') } + scope :pending, -> { where(status: 'pending') } + scope :success, -> { where(status: 'success') } + scope :failed, -> { where(status: 'failed') } + scope :canceled, -> { where(status: 'canceled') } + scope :skipped, -> { where(status: 'skipped') } + scope :manual, -> { where(status: 'manual') } + + scope :failed_but_allowed, -> do + where(allow_failure: true, status: [:failed, :canceled]) + end + + scope :exclude_ignored, -> do + where("allow_failure = ? OR status IN (?)", + false, %w[created pending running success skipped]) + end + + def self.status_sql + scope_relevant = latest.exclude_ignored + scope_warnings = latest.failed_but_allowed + + builds = scope_relevant.select('count(*)').to_sql + created = scope_relevant.created.select('count(*)').to_sql + success = scope_relevant.success.select('count(*)').to_sql + manual = scope_relevant.manual.select('count(*)').to_sql + pending = scope_relevant.pending.select('count(*)').to_sql + running = scope_relevant.running.select('count(*)').to_sql + skipped = scope_relevant.skipped.select('count(*)').to_sql + canceled = scope_relevant.canceled.select('count(*)').to_sql + warnings = scope_warnings.select('count(*) > 0').to_sql + + <<-SQL.strip_heredoc + (CASE + WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]} + WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]} + WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]} + WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]} + WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]} + WHEN (#{created}) > 0 THEN #{STATUSES[:running]} + ELSE #{STATUSES[:failed]} + END) + SQL + end + end + + def perform(id) + status_sql = Build + .where('ci_builds.commit_id = ci_stages.pipeline_id') + .where('ci_builds.stage = ci_stages.name') + .status_sql + + sql = <<-SQL + UPDATE ci_stages SET status = (#{status_sql}) + WHERE ci_stages.id = #{id.to_i} + SQL + + ActiveRecord::Base.connection.execute(sql) + end + end + end +end diff --git a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb index a32a7fceb68..ff137cc7d47 100644 --- a/spec/migrations/migrate_stage_id_reference_in_background_spec.rb +++ b/spec/migrations/migrate_stage_id_reference_in_background_spec.rb @@ -2,19 +2,6 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20170628080858_migrate_stage_id_reference_in_background') describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do - matcher :be_scheduled_migration do |delay, *expected| - match do |migration| - BackgroundMigrationWorker.jobs.any? do |job| - job['args'] == [migration, expected] && - job['at'].to_i == (delay.to_i + Time.now.to_i) - end - end - - failure_message do |migration| - "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!" - end - end - let(:jobs) { table(:ci_builds) } let(:stages) { table(:ci_stages) } let(:pipelines) { table(:ci_pipelines) } diff --git a/spec/migrations/migrate_stages_statuses_spec.rb b/spec/migrations/migrate_stages_statuses_spec.rb index 478ddad262a..8463583cef3 100644 --- a/spec/migrations/migrate_stages_statuses_spec.rb +++ b/spec/migrations/migrate_stages_statuses_spec.rb @@ -11,6 +11,8 @@ describe MigrateStagesStatuses, :migration do failed: 4, canceled: 5, skipped: 6, manual: 7 } before do + stub_const("#{described_class.name}::BATCH_SIZE", 2) + projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1') projects.create!(id: 2, name: 'gitlab2', path: 'gitlab2') @@ -31,15 +33,31 @@ describe MigrateStagesStatuses, :migration do end it 'correctly migrates stages statuses' do - expect(stages.where(status: nil).count).to eq 3 + Sidekiq::Testing.inline! do + expect(stages.where(status: nil).count).to eq 3 - migrate! + migrate! - expect(stages.where(status: nil)).to be_empty - expect(stages.all.order('id ASC').pluck(:status)) - .to eq [STATUSES[:running], STATUSES[:failed], STATUSES[:success]] + expect(stages.where(status: nil)).to be_empty + expect(stages.all.order('id ASC').pluck(:status)) + .to eq [STATUSES[:running], STATUSES[:failed], STATUSES[:success]] + end end + it 'correctly schedules background migrations' do + Sidekiq::Testing.fake! do + Timecop.freeze do + migrate! + + expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 1) + expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 2) + expect(described_class::MIGRATION).to be_scheduled_migration(10.minutes, 3) + expect(BackgroundMigrationWorker.jobs.size).to eq 3 + end + end + end + + def create_job(project:, pipeline:, stage:, status:, **opts) stages = { test: 1, build: 2, deploy: 3} diff --git a/spec/support/background_migrations_matchers.rb b/spec/support/background_migrations_matchers.rb new file mode 100644 index 00000000000..423c0e4cefc --- /dev/null +++ b/spec/support/background_migrations_matchers.rb @@ -0,0 +1,13 @@ +RSpec::Matchers.define :be_scheduled_migration do |delay, *expected| + match do |migration| + BackgroundMigrationWorker.jobs.any? do |job| + job['args'] == [migration, expected] && + job['at'].to_i == (delay.to_i + Time.now.to_i) + end + end + + failure_message do |migration| + "Migration `#{migration}` with args `#{expected.inspect}` " \ + 'not scheduled in expected time!' + end +end -- cgit v1.2.1 From 7082530d555ad98fede2823d2123622abaf1c3a3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 10 Jul 2017 15:42:19 +0200 Subject: Schedule stages statuses bg migrations in batches --- db/post_migrate/20170630111158_migrate_stages_statuses.rb | 15 ++++++++------- lib/gitlab/background_migration/migrate_stage_status.rb | 5 +++-- spec/migrations/migrate_stages_statuses_spec.rb | 8 +++++--- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index 2bc067e5d90..1641e550480 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -6,21 +6,22 @@ class MigrateStagesStatuses < ActiveRecord::Migration disable_ddl_transaction! BATCH_SIZE = 10000 + RANGE_SIZE = 1000 MIGRATION = 'MigrateStageStatus'.freeze class Stage < ActiveRecord::Base self.table_name = 'ci_stages' + include ::EachBatch end def up - index = 1 + Stage.where(status: nil).each_batch(of: BATCH_SIZE) do |relation, index| + relation.each_batch(of: RANGE_SIZE) do |batch| + range = relation.pluck('MIN(id)', 'MAX(id)').first + schedule = index * 5.minutes - Stage.where(status: nil).in_batches(of: BATCH_SIZE) do |relation| - jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } - schedule = index * 5.minutes - index += 1 - - BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) + BackgroundMigrationWorker.perform_in(schedule, MIGRATION, range) + end end end diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb index e4fdc723b13..3c9744d1607 100644 --- a/lib/gitlab/background_migration/migrate_stage_status.rb +++ b/lib/gitlab/background_migration/migrate_stage_status.rb @@ -58,7 +58,7 @@ module Gitlab end end - def perform(id) + def perform(start_id, stop_id) status_sql = Build .where('ci_builds.commit_id = ci_stages.pipeline_id') .where('ci_builds.stage = ci_stages.name') @@ -66,7 +66,8 @@ module Gitlab sql = <<-SQL UPDATE ci_stages SET status = (#{status_sql}) - WHERE ci_stages.id = #{id.to_i} + WHERE ci_stages.status IS NULL + AND ci_stages.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} SQL ActiveRecord::Base.connection.execute(sql) diff --git a/spec/migrations/migrate_stages_statuses_spec.rb b/spec/migrations/migrate_stages_statuses_spec.rb index 8463583cef3..1769b1e256b 100644 --- a/spec/migrations/migrate_stages_statuses_spec.rb +++ b/spec/migrations/migrate_stages_statuses_spec.rb @@ -12,6 +12,7 @@ describe MigrateStagesStatuses, :migration do before do stub_const("#{described_class.name}::BATCH_SIZE", 2) + stub_const("#{described_class.name}::RANGE_SIZE", 1) projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1') projects.create!(id: 2, name: 'gitlab2', path: 'gitlab2') @@ -49,9 +50,10 @@ describe MigrateStagesStatuses, :migration do Timecop.freeze do migrate! - expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 1) - expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 2) - expect(described_class::MIGRATION).to be_scheduled_migration(10.minutes, 3) + puts BackgroundMigrationWorker.jobs.inspect + expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 1, 1) + expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 2, 2) + expect(described_class::MIGRATION).to be_scheduled_migration(10.minutes, 3, 3) expect(BackgroundMigrationWorker.jobs.size).to eq 3 end end -- cgit v1.2.1 From 6e9924a2245004c6b6adb34028880a46fd5471df Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 11 Jul 2017 11:56:15 +0200 Subject: Add a new stage status column to safe attributes --- spec/lib/gitlab/import_export/safe_model_attributes.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 4ef3db3721f..62163c54c6d 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -219,6 +219,7 @@ Ci::Pipeline: Ci::Stage: - id - name +- status - project_id - pipeline_id - created_at -- cgit v1.2.1 From 1d087e073660130c81bf0917a6fa395886b6e2dc Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 11 Jul 2017 12:01:35 +0200 Subject: Freeze mutable constants in stages migration code --- app/models/concerns/has_status.rb | 2 +- lib/gitlab/background_migration/migrate_stage_status.rb | 2 +- spec/migrations/migrate_stages_statuses_spec.rb | 5 ++--- spec/models/commit_status_spec.rb | 1 - 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 8ea5a007f76..a94226bb735 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -9,7 +9,7 @@ module HasStatus COMPLETED_STATUSES = %w[success failed canceled skipped].freeze ORDERED_STATUSES = %w[failed pending running manual canceled success skipped created].freeze STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7 } + failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze class_methods do def enumerated_status! diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb index 3c9744d1607..b1ff0900709 100644 --- a/lib/gitlab/background_migration/migrate_stage_status.rb +++ b/lib/gitlab/background_migration/migrate_stage_status.rb @@ -2,7 +2,7 @@ module Gitlab module BackgroundMigration class MigrateStageStatus STATUSES = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7 } + failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze class Build < ActiveRecord::Base self.table_name = 'ci_builds' diff --git a/spec/migrations/migrate_stages_statuses_spec.rb b/spec/migrations/migrate_stages_statuses_spec.rb index 1769b1e256b..7a424bdaedb 100644 --- a/spec/migrations/migrate_stages_statuses_spec.rb +++ b/spec/migrations/migrate_stages_statuses_spec.rb @@ -8,7 +8,7 @@ describe MigrateStagesStatuses, :migration do let(:projects) { table(:projects) } STATUSES = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7 } + failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze before do stub_const("#{described_class.name}::BATCH_SIZE", 2) @@ -59,9 +59,8 @@ describe MigrateStagesStatuses, :migration do end end - def create_job(project:, pipeline:, stage:, status:, **opts) - stages = { test: 1, build: 2, deploy: 3} + stages = { test: 1, build: 2, deploy: 3 } jobs.create!(project_id: project, commit_id: pipeline, stage_idx: stages[stage.to_sym], stage: stage, diff --git a/spec/models/commit_status_spec.rb b/spec/models/commit_status_spec.rb index c0cbf0b2f95..c7651ce9b46 100644 --- a/spec/models/commit_status_spec.rb +++ b/spec/models/commit_status_spec.rb @@ -419,7 +419,6 @@ describe CommitStatus, :models do end end - describe '#locking_enabled?' do before do commit_status.lock_version = 100 -- cgit v1.2.1 From 65b3c220090d8fbfcfb2c4ba3b4d70d3c30fd7e3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 11 Jul 2017 12:28:04 +0200 Subject: Fix pipeline stages statuses migration specs --- spec/migrations/migrate_stages_statuses_spec.rb | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/spec/migrations/migrate_stages_statuses_spec.rb b/spec/migrations/migrate_stages_statuses_spec.rb index 7a424bdaedb..ace1efa44ba 100644 --- a/spec/migrations/migrate_stages_statuses_spec.rb +++ b/spec/migrations/migrate_stages_statuses_spec.rb @@ -12,7 +12,7 @@ describe MigrateStagesStatuses, :migration do before do stub_const("#{described_class.name}::BATCH_SIZE", 2) - stub_const("#{described_class.name}::RANGE_SIZE", 1) + stub_const("#{described_class.name}::RANGE_SIZE", 2) projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1') projects.create!(id: 2, name: 'gitlab2', path: 'gitlab2') @@ -50,11 +50,9 @@ describe MigrateStagesStatuses, :migration do Timecop.freeze do migrate! - puts BackgroundMigrationWorker.jobs.inspect - expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 1, 1) - expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 2, 2) + expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 1, 2) expect(described_class::MIGRATION).to be_scheduled_migration(10.minutes, 3, 3) - expect(BackgroundMigrationWorker.jobs.size).to eq 3 + expect(BackgroundMigrationWorker.jobs.size).to eq 2 end end end -- cgit v1.2.1 From bb67b4749b5b4c62d4235c90dc0320967f850cdd Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 11 Jul 2017 14:31:04 +0200 Subject: Update version number of stages statuses migration --- .../20170630105320_add_status_to_ci_stages.rb | 9 ------ .../20170711145320_add_status_to_ci_stages.rb | 9 ++++++ .../20170630111158_migrate_stages_statuses.rb | 35 ---------------------- .../20170711145558_migrate_stages_statuses.rb | 35 ++++++++++++++++++++++ db/schema.rb | 2 +- spec/migrations/migrate_stages_statuses_spec.rb | 2 +- 6 files changed, 46 insertions(+), 46 deletions(-) delete mode 100644 db/migrate/20170630105320_add_status_to_ci_stages.rb create mode 100644 db/migrate/20170711145320_add_status_to_ci_stages.rb delete mode 100644 db/post_migrate/20170630111158_migrate_stages_statuses.rb create mode 100644 db/post_migrate/20170711145558_migrate_stages_statuses.rb diff --git a/db/migrate/20170630105320_add_status_to_ci_stages.rb b/db/migrate/20170630105320_add_status_to_ci_stages.rb deleted file mode 100644 index d497a61a959..00000000000 --- a/db/migrate/20170630105320_add_status_to_ci_stages.rb +++ /dev/null @@ -1,9 +0,0 @@ -class AddStatusToCiStages < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - def change - add_column :ci_stages, :status, :integer - end -end diff --git a/db/migrate/20170711145320_add_status_to_ci_stages.rb b/db/migrate/20170711145320_add_status_to_ci_stages.rb new file mode 100644 index 00000000000..d497a61a959 --- /dev/null +++ b/db/migrate/20170711145320_add_status_to_ci_stages.rb @@ -0,0 +1,9 @@ +class AddStatusToCiStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :ci_stages, :status, :integer + end +end diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb deleted file mode 100644 index 1641e550480..00000000000 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ /dev/null @@ -1,35 +0,0 @@ -class MigrateStagesStatuses < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - disable_ddl_transaction! - - BATCH_SIZE = 10000 - RANGE_SIZE = 1000 - MIGRATION = 'MigrateStageStatus'.freeze - - class Stage < ActiveRecord::Base - self.table_name = 'ci_stages' - include ::EachBatch - end - - def up - Stage.where(status: nil).each_batch(of: BATCH_SIZE) do |relation, index| - relation.each_batch(of: RANGE_SIZE) do |batch| - range = relation.pluck('MIN(id)', 'MAX(id)').first - schedule = index * 5.minutes - - BackgroundMigrationWorker.perform_in(schedule, MIGRATION, range) - end - end - end - - def down - disable_statement_timeout - - execute <<-SQL.strip_heredoc - UPDATE ci_stages SET status = null - SQL - end -end diff --git a/db/post_migrate/20170711145558_migrate_stages_statuses.rb b/db/post_migrate/20170711145558_migrate_stages_statuses.rb new file mode 100644 index 00000000000..1641e550480 --- /dev/null +++ b/db/post_migrate/20170711145558_migrate_stages_statuses.rb @@ -0,0 +1,35 @@ +class MigrateStagesStatuses < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + BATCH_SIZE = 10000 + RANGE_SIZE = 1000 + MIGRATION = 'MigrateStageStatus'.freeze + + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' + include ::EachBatch + end + + def up + Stage.where(status: nil).each_batch(of: BATCH_SIZE) do |relation, index| + relation.each_batch(of: RANGE_SIZE) do |batch| + range = relation.pluck('MIN(id)', 'MAX(id)').first + schedule = index * 5.minutes + + BackgroundMigrationWorker.perform_in(schedule, MIGRATION, range) + end + end + end + + def down + disable_statement_timeout + + execute <<-SQL.strip_heredoc + UPDATE ci_stages SET status = null + SQL + end +end diff --git a/db/schema.rb b/db/schema.rb index e0c9e5efb33..3ef311f48d8 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170707184244) do +ActiveRecord::Schema.define(version: 20170711145558) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/spec/migrations/migrate_stages_statuses_spec.rb b/spec/migrations/migrate_stages_statuses_spec.rb index ace1efa44ba..4102d57e368 100644 --- a/spec/migrations/migrate_stages_statuses_spec.rb +++ b/spec/migrations/migrate_stages_statuses_spec.rb @@ -1,5 +1,5 @@ require 'spec_helper' -require Rails.root.join('db', 'post_migrate', '20170630111158_migrate_stages_statuses.rb') +require Rails.root.join('db', 'post_migrate', '20170711145558_migrate_stages_statuses.rb') describe MigrateStagesStatuses, :migration do let(:jobs) { table(:ci_builds) } -- cgit v1.2.1 From c7a7ef044cf79dcd5ffd25b9fb325cd0abd612b2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 20 Jul 2017 12:56:27 +0200 Subject: Use a new stage_id reference to a persisted stage --- app/models/commit_status.rb | 4 +--- spec/models/commit_status_spec.rb | 7 ++++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index afdc75f75fb..efb5cbd9d41 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -43,9 +43,7 @@ class CommitStatus < ActiveRecord::Base # `ci_builds.stage` attribute and migrate `ci_builds.stage_id` reference in # one of upcoming releases. # - def stage_entity - Ci::Stage.find_by(pipeline: pipeline, name: stage) - end + belongs_to :stage_entity, foreign_key: :stage_id, class_name: 'Ci::Stage' state_machine :status do event :enqueue do diff --git a/spec/models/commit_status_spec.rb b/spec/models/commit_status_spec.rb index c7651ce9b46..f54e1131813 100644 --- a/spec/models/commit_status_spec.rb +++ b/spec/models/commit_status_spec.rb @@ -409,9 +409,10 @@ describe CommitStatus, :models do end describe '#stage_entity' do - let!(:stage) do - create(:ci_stage_entity, pipeline: commit_status.pipeline, - name: commit_status.stage) + let(:stage) { create(:ci_stage_entity) } + + let(:commit_status) do + create(:commit_status, stage_id: stage.id) end it 'has a correct association with persisted stage' do -- cgit v1.2.1 From 0605cdd7590b12bad073bf41f3e793274e931a80 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 20 Jul 2017 13:05:26 +0200 Subject: Implement proper associations with a persisted stage --- app/models/ci/stage.rb | 4 ++-- spec/models/ci/stage_spec.rb | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index da1c3753924..ca89caf4782 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -9,8 +9,8 @@ module Ci belongs_to :project belongs_to :pipeline - has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id - has_many :builds, foreign_key: :commit_id + has_many :commit_statuses, foreign_key: :stage_id + has_many :builds, foreign_key: :stage_id validates :project, presence: true, unless: :importing? validates :pipeline, presence: true, unless: :importing? diff --git a/spec/models/ci/stage_spec.rb b/spec/models/ci/stage_spec.rb index 911c468ff1a..49573175266 100644 --- a/spec/models/ci/stage_spec.rb +++ b/spec/models/ci/stage_spec.rb @@ -1,6 +1,27 @@ require 'spec_helper' describe Ci::Stage, :models do + describe 'associations' do + let(:stage) { create(:ci_stage_entity) } + + before do + create(:ci_build, stage_id: stage.id) + create(:commit_status, stage_id: stage.id) + end + + describe '#commit_statuses' do + it 'returns all commit statuses' do + expect(stage.commit_statuses.count).to be 2 + end + end + + describe '#builds' do + it 'returns only builds' do + expect(stage.builds).to be_one + end + end + end + describe '#status' do context 'when stage is pending' do let(:stage) { create(:ci_stage_entity, status: 'pending') } -- cgit v1.2.1 From 470661e1a70bd3c8415387e9068823536d1fb1bc Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 20 Jul 2017 13:06:45 +0200 Subject: Change a method name which enumerates CI/CD statuses --- app/models/ci/stage.rb | 2 +- app/models/concerns/has_status.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index ca89caf4782..c163d047127 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -4,7 +4,7 @@ module Ci include Importable include HasStatus - enumerated_status! + enumerate_status! belongs_to :project belongs_to :pipeline diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index a94226bb735..758d71b7f4c 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -12,7 +12,7 @@ module HasStatus failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze class_methods do - def enumerated_status! + def enumerate_status! enum status: HasStatus::STATUSES_ENUM end -- cgit v1.2.1 From e389507650769304bd61f7a82431cc6c07feb364 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 20 Jul 2017 13:17:48 +0200 Subject: Add optimistic locking column to ci_stages table --- db/migrate/20170720111708_add_lock_version_to_ci_stages.rb | 9 +++++++++ db/schema.rb | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20170720111708_add_lock_version_to_ci_stages.rb diff --git a/db/migrate/20170720111708_add_lock_version_to_ci_stages.rb b/db/migrate/20170720111708_add_lock_version_to_ci_stages.rb new file mode 100644 index 00000000000..e1c4f033286 --- /dev/null +++ b/db/migrate/20170720111708_add_lock_version_to_ci_stages.rb @@ -0,0 +1,9 @@ +class AddLockVersionToCiStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :ci_stages, :lock_version, :integer + end +end diff --git a/db/schema.rb b/db/schema.rb index 862b2e21f4d..567ba4d061b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170717150329) do +ActiveRecord::Schema.define(version: 20170720111708) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -366,6 +366,7 @@ ActiveRecord::Schema.define(version: 20170717150329) do t.datetime "updated_at" t.string "name" t.integer "status" + t.integer "lock_version" end add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree -- cgit v1.2.1 From bbdc35717c1ba08630f5b2ae59a333a81941b181 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 20 Jul 2017 13:30:05 +0200 Subject: Implement method that updates a stage status --- app/models/ci/stage.rb | 47 ++++++++++++++++++++++++++++++++++++++++++++ spec/models/ci/stage_spec.rb | 22 +++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index c163d047127..066903ddc5b 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -3,6 +3,7 @@ module Ci extend Ci::Model include Importable include HasStatus + include Gitlab::OptimisticLocking enumerate_status! @@ -15,5 +16,51 @@ module Ci validates :project, presence: true, unless: :importing? validates :pipeline, presence: true, unless: :importing? validates :name, presence: true, unless: :importing? + + state_machine :status, initial: :created do + event :enqueue do + transition created: :pending + transition [:success, :failed, :canceled, :skipped] => :running + end + + event :run do + transition any - [:running] => :running + end + + event :skip do + transition any - [:skipped] => :skipped + end + + event :drop do + transition any - [:failed] => :failed + end + + event :succeed do + transition any - [:success] => :success + end + + event :cancel do + transition any - [:canceled] => :canceled + end + + event :block do + transition any - [:manual] => :manual + end + end + + def update! + retry_optimistic_lock(self) do + case commit_statuses.latest.status + when 'pending' then enqueue + when 'running' then run + when 'success' then succeed + when 'failed' then drop + when 'canceled' then cancel + when 'manual' then block + when 'skipped' then skip + else skip + end + end + end end end diff --git a/spec/models/ci/stage_spec.rb b/spec/models/ci/stage_spec.rb index 49573175266..e829ccb048e 100644 --- a/spec/models/ci/stage_spec.rb +++ b/spec/models/ci/stage_spec.rb @@ -1,9 +1,9 @@ require 'spec_helper' describe Ci::Stage, :models do - describe 'associations' do - let(:stage) { create(:ci_stage_entity) } + let(:stage) { create(:ci_stage_entity) } + describe 'associations' do before do create(:ci_build, stage_id: stage.id) create(:commit_status, stage_id: stage.id) @@ -39,4 +39,22 @@ describe Ci::Stage, :models do end end end + + describe 'update!' do + context 'when stage objects needs to be updated' do + before do + create(:ci_build, :success, stage_id: stage.id) + create(:ci_build, :running, stage_id: stage.id) + end + + it 'updates stage status correctly' do + expect { stage.update! } + .to change { stage.reload.status } + .to 'running' + end + end + + context 'when stage object is locked' do + end + end end -- cgit v1.2.1 From 5505795ed2e65da20a896b67422a075515552a35 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 20 Jul 2017 13:30:58 +0200 Subject: Add simple asynchronous stage update worker --- app/workers/stage_update_worker.rb | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 app/workers/stage_update_worker.rb diff --git a/app/workers/stage_update_worker.rb b/app/workers/stage_update_worker.rb new file mode 100644 index 00000000000..1d6f23305d6 --- /dev/null +++ b/app/workers/stage_update_worker.rb @@ -0,0 +1,8 @@ +class StageUpdateWorker + include Sidekiq::Worker + include PipelineQueue + + def perform(stage_id) + Ci::Stage.find_by(id: stage_id)&.update! + end +end -- cgit v1.2.1 From d3814ad69876c69cdab574e2958368f2ab648171 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 24 Jul 2017 11:22:01 +0200 Subject: Adds some specs for stage optimistic locking --- .../gitlab/import_export/safe_model_attributes.yml | 1 + spec/models/ci/stage_spec.rb | 19 +++++++++++++++++++ spec/services/ci/retry_build_service_spec.rb | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 62163c54c6d..7bbeea06f04 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -220,6 +220,7 @@ Ci::Stage: - id - name - status +- lock_version - project_id - pipeline_id - created_at diff --git a/spec/models/ci/stage_spec.rb b/spec/models/ci/stage_spec.rb index e829ccb048e..3dc00017a2c 100644 --- a/spec/models/ci/stage_spec.rb +++ b/spec/models/ci/stage_spec.rb @@ -54,7 +54,26 @@ describe Ci::Stage, :models do end end + context 'when stage is skipped' do + it 'updates status to skipped' do + expect { stage.update! } + .to change { stage.reload.status } + .to 'skipped' + end + end + context 'when stage object is locked' do + before do + create(:ci_build, :failed, stage_id: stage.id) + end + + it 'retries a lock to update a stage status' do + stage.lock_version = 100 + + stage.update! + + expect(stage.reload).to be_failed + end end end end diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index ef9927c5969..a0e83deeb54 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -22,7 +22,7 @@ describe Ci::RetryBuildService, :services do %i[type lock_version target_url base_tags commit_id deployments erased_by_id last_deployment project_id runner_id tag_taggings taggings tags trigger_request_id - user_id auto_canceled_by_id retried].freeze + user_id auto_canceled_by_id retried stage_entity].freeze shared_examples 'build duplication' do let(:stage) do -- cgit v1.2.1 From 8657d5dd8af6c365b41d7c2997e6e5c9e18e8241 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 24 Jul 2017 11:33:01 +0200 Subject: Do not implement CI/CD job to stage association yet --- app/models/commit_status.rb | 15 ++++----------- spec/models/commit_status_spec.rb | 12 ------------ spec/services/ci/retry_build_service_spec.rb | 2 +- 3 files changed, 5 insertions(+), 24 deletions(-) diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index efb5cbd9d41..d7418a07177 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -38,22 +38,15 @@ class CommitStatus < ActiveRecord::Base scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) } scope :after_stage, -> (index) { where('stage_idx > ?', index) } - ## - # TODO, we will change this to `belongs_to :stage` when we phase out - # `ci_builds.stage` attribute and migrate `ci_builds.stage_id` reference in - # one of upcoming releases. - # - belongs_to :stage_entity, foreign_key: :stage_id, class_name: 'Ci::Stage' - state_machine :status do - event :enqueue do - transition [:created, :skipped, :manual] => :pending - end - event :process do transition [:skipped, :manual] => :created end + event :enqueue do + transition [:created, :skipped, :manual] => :pending + end + event :run do transition pending: :running end diff --git a/spec/models/commit_status_spec.rb b/spec/models/commit_status_spec.rb index f54e1131813..f173e2c142a 100644 --- a/spec/models/commit_status_spec.rb +++ b/spec/models/commit_status_spec.rb @@ -408,18 +408,6 @@ describe CommitStatus, :models do end end - describe '#stage_entity' do - let(:stage) { create(:ci_stage_entity) } - - let(:commit_status) do - create(:commit_status, stage_id: stage.id) - end - - it 'has a correct association with persisted stage' do - expect(commit_status.stage_entity).to eq stage - end - end - describe '#locking_enabled?' do before do commit_status.lock_version = 100 diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index a0e83deeb54..ef9927c5969 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -22,7 +22,7 @@ describe Ci::RetryBuildService, :services do %i[type lock_version target_url base_tags commit_id deployments erased_by_id last_deployment project_id runner_id tag_taggings taggings tags trigger_request_id - user_id auto_canceled_by_id retried stage_entity].freeze + user_id auto_canceled_by_id retried].freeze shared_examples 'build duplication' do let(:stage) do -- cgit v1.2.1 From 865de49b0832dc1e2fa74034a25d186980b6a361 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 24 Jul 2017 11:34:01 +0200 Subject: Update related stage status when job status is changed --- app/models/ci/stage.rb | 2 ++ app/models/commit_status.rb | 1 + 2 files changed, 3 insertions(+) diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 066903ddc5b..78caa70e52e 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -17,6 +17,8 @@ module Ci validates :pipeline, presence: true, unless: :importing? validates :name, presence: true, unless: :importing? + ## TODO, should we extract these events to `Ci::Eventable`? + # state_machine :status, initial: :created do event :enqueue do transition created: :pending diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index d7418a07177..842c6e5cb50 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -91,6 +91,7 @@ class CommitStatus < ActiveRecord::Base end end + StageUpdateWorker.perform_async(commit_status.stage_id) ExpireJobCacheWorker.perform_async(commit_status.id) end end -- cgit v1.2.1 From c14bd53d7483b57d47159c52dcb1b3456885f153 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 24 Jul 2017 14:13:29 +0200 Subject: Fix import/export for CI/CD stage commit statuses --- spec/lib/gitlab/import_export/all_models.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 977174a5fd2..28df089928f 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -115,7 +115,7 @@ pipelines: stages: - project - pipeline -- statuses +- commit_statuses - builds statuses: - project -- cgit v1.2.1 From f4e01b597c7cb40797d0b690451322bc79d8dfe0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 26 Jul 2017 14:48:32 +0200 Subject: Move enum specific code from a concern to CI stage --- app/models/ci/stage.rb | 2 +- app/models/concerns/has_status.rb | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 78caa70e52e..b1cca06abaa 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -5,7 +5,7 @@ module Ci include HasStatus include Gitlab::OptimisticLocking - enumerate_status! + enum status: HasStatus::STATUSES_ENUM belongs_to :project belongs_to :pipeline diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 758d71b7f4c..3803e18a96e 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -12,10 +12,6 @@ module HasStatus failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze class_methods do - def enumerate_status! - enum status: HasStatus::STATUSES_ENUM - end - def status_sql scope_relevant = respond_to?(:exclude_ignored) ? exclude_ignored : all scope_warnings = respond_to?(:failed_but_allowed) ? failed_but_allowed : none -- cgit v1.2.1 From 7d6538f2e2a6f1b0808a77e347a9083295b17c8c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 26 Jul 2017 14:50:48 +0200 Subject: Rename method responsible for updating stage status --- app/models/ci/stage.rb | 2 +- app/workers/stage_update_worker.rb | 4 +++- spec/models/ci/stage_spec.rb | 8 ++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index b1cca06abaa..cd2f1dd3509 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -50,7 +50,7 @@ module Ci end end - def update! + def update_status retry_optimistic_lock(self) do case commit_statuses.latest.status when 'pending' then enqueue diff --git a/app/workers/stage_update_worker.rb b/app/workers/stage_update_worker.rb index 1d6f23305d6..eef0b11e70b 100644 --- a/app/workers/stage_update_worker.rb +++ b/app/workers/stage_update_worker.rb @@ -3,6 +3,8 @@ class StageUpdateWorker include PipelineQueue def perform(stage_id) - Ci::Stage.find_by(id: stage_id)&.update! + Ci::Stage.find_by(id: stage_id).try do |stage| + stage.update_status + end end end diff --git a/spec/models/ci/stage_spec.rb b/spec/models/ci/stage_spec.rb index 3dc00017a2c..d5c66598451 100644 --- a/spec/models/ci/stage_spec.rb +++ b/spec/models/ci/stage_spec.rb @@ -40,7 +40,7 @@ describe Ci::Stage, :models do end end - describe 'update!' do + describe 'update_status' do context 'when stage objects needs to be updated' do before do create(:ci_build, :success, stage_id: stage.id) @@ -48,7 +48,7 @@ describe Ci::Stage, :models do end it 'updates stage status correctly' do - expect { stage.update! } + expect { stage.update_status } .to change { stage.reload.status } .to 'running' end @@ -56,7 +56,7 @@ describe Ci::Stage, :models do context 'when stage is skipped' do it 'updates status to skipped' do - expect { stage.update! } + expect { stage.update_status } .to change { stage.reload.status } .to 'skipped' end @@ -70,7 +70,7 @@ describe Ci::Stage, :models do it 'retries a lock to update a stage status' do stage.lock_version = 100 - stage.update! + stage.update_status expect(stage.reload).to be_failed end -- cgit v1.2.1 From cec2cc3ffa3077346c81625f6b1f48c113e19e93 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 31 Jul 2017 12:07:05 +0200 Subject: Add specs for stage update worker --- spec/workers/stage_update_worker_spec.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 spec/workers/stage_update_worker_spec.rb diff --git a/spec/workers/stage_update_worker_spec.rb b/spec/workers/stage_update_worker_spec.rb new file mode 100644 index 00000000000..7bc76c79464 --- /dev/null +++ b/spec/workers/stage_update_worker_spec.rb @@ -0,0 +1,22 @@ +require 'spec_helper' + +describe StageUpdateWorker do + describe '#perform' do + context 'when stage exists' do + let(:stage) { create(:ci_stage_entity) } + + it 'updates stage status' do + expect_any_instance_of(Ci::Stage).to receive(:update_status) + + described_class.new.perform(stage.id) + end + end + + context 'when stage does not exist' do + it 'does not raise exception' do + expect { described_class.new.perform(123) } + .not_to raise_error + end + end + end +end -- cgit v1.2.1 From 13a15e7009e292919109ea911640a627dbd0e327 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 31 Jul 2017 12:23:04 +0200 Subject: Use update_column_in_batches helper in stages migration --- app/models/ci/stage.rb | 2 -- db/post_migrate/20170711145558_migrate_stages_statuses.rb | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index cd2f1dd3509..7819bc3cd2c 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -17,8 +17,6 @@ module Ci validates :pipeline, presence: true, unless: :importing? validates :name, presence: true, unless: :importing? - ## TODO, should we extract these events to `Ci::Eventable`? - # state_machine :status, initial: :created do event :enqueue do transition created: :pending diff --git a/db/post_migrate/20170711145558_migrate_stages_statuses.rb b/db/post_migrate/20170711145558_migrate_stages_statuses.rb index 1641e550480..5a24fb1307f 100644 --- a/db/post_migrate/20170711145558_migrate_stages_statuses.rb +++ b/db/post_migrate/20170711145558_migrate_stages_statuses.rb @@ -28,8 +28,6 @@ class MigrateStagesStatuses < ActiveRecord::Migration def down disable_statement_timeout - execute <<-SQL.strip_heredoc - UPDATE ci_stages SET status = null - SQL + update_column_in_batches(:ci_stages, :status, nil) end end -- cgit v1.2.1 From f6d75f091733aa94dcfe011c374d885be4ad6c11 Mon Sep 17 00:00:00 2001 From: haseeb Date: Thu, 10 Aug 2017 20:38:24 +0530 Subject: copy milestones from issue when using create merge request btn --- app/controllers/projects/issues_controller.rb | 2 +- app/services/merge_requests/create_from_issue_service.rb | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index f4d4cca8dd8..8893a514207 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -212,7 +212,7 @@ class Projects::IssuesController < Projects::ApplicationController end def create_merge_request - result = MergeRequests::CreateFromIssueService.new(project, current_user, issue_iid: issue.iid).execute + result = ::MergeRequests::CreateFromIssueService.new(project, current_user, issue_iid: issue.iid).execute if result[:status] == :success render json: MergeRequestCreateSerializer.new.represent(result[:merge_request]) diff --git a/app/services/merge_requests/create_from_issue_service.rb b/app/services/merge_requests/create_from_issue_service.rb index 738cedbaed7..aec68c74aa6 100644 --- a/app/services/merge_requests/create_from_issue_service.rb +++ b/app/services/merge_requests/create_from_issue_service.rb @@ -43,7 +43,9 @@ module MergeRequests { source_project_id: project.id, source_branch: branch_name, - target_project_id: project.id + target_project_id: project.id, + labels: issue.labels, + milestone_id: issue.milestone_id } end -- cgit v1.2.1 From a115267e25f8f2499156ea04bf1d9c0296b09270 Mon Sep 17 00:00:00 2001 From: haseeb Date: Thu, 10 Aug 2017 23:05:20 +0530 Subject: working version of create merge requests with labels --- app/controllers/projects/issues_controller.rb | 12 +++++++++++- app/services/merge_requests/create_from_issue_service.rb | 4 +--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index 8893a514207..04c14806544 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -212,7 +212,7 @@ class Projects::IssuesController < Projects::ApplicationController end def create_merge_request - result = ::MergeRequests::CreateFromIssueService.new(project, current_user, issue_iid: issue.iid).execute + result = ::MergeRequests::CreateFromIssueService.new(project, current_user, merge_request_params).execute if result[:status] == :success render json: MergeRequestCreateSerializer.new.represent(result[:merge_request]) @@ -287,4 +287,14 @@ class Projects::IssuesController < Projects::ApplicationController redirect_to new_user_session_path, notice: notice end + + private + + def merge_request_params + { + issue_iid: issue.iid, + label_ids: issue.label_ids, + milestone_id: issue.milestone_id + } + end end diff --git a/app/services/merge_requests/create_from_issue_service.rb b/app/services/merge_requests/create_from_issue_service.rb index aec68c74aa6..738cedbaed7 100644 --- a/app/services/merge_requests/create_from_issue_service.rb +++ b/app/services/merge_requests/create_from_issue_service.rb @@ -43,9 +43,7 @@ module MergeRequests { source_project_id: project.id, source_branch: branch_name, - target_project_id: project.id, - labels: issue.labels, - milestone_id: issue.milestone_id + target_project_id: project.id } end -- cgit v1.2.1 From 59d7ffad06828435e31c7ea09c97c4b79909ef77 Mon Sep 17 00:00:00 2001 From: haseeb Date: Fri, 11 Aug 2017 00:14:15 +0530 Subject: tests for inheriting milestones and labels --- .../create_from_issue_service_spec.rb | 27 +++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/spec/services/merge_requests/create_from_issue_service_spec.rb b/spec/services/merge_requests/create_from_issue_service_spec.rb index 492b55cdece..bdc53c7e64b 100644 --- a/spec/services/merge_requests/create_from_issue_service_spec.rb +++ b/spec/services/merge_requests/create_from_issue_service_spec.rb @@ -4,8 +4,21 @@ describe MergeRequests::CreateFromIssueService do let(:project) { create(:project, :repository) } let(:user) { create(:user) } let(:issue) { create(:issue, project: project) } - + let(:milestone) { create(:milestone, project: project) } + let(:labels) { create_pair(:label, project: project) } + subject(:service) { described_class.new(project, user, issue_iid: issue.iid) } + subject(:service2) do + described_class.new( + project, + user, + { + issue_iid: issue.iid, + milestone_id: milestone.id, + label_ids: labels.map(&:id) + } + ) + end before do project.add_developer(user) @@ -25,6 +38,18 @@ describe MergeRequests::CreateFromIssueService do described_class.new(project, user, issue_iid: -1).execute end + it "can inherit labels" do + result = service2.execute + + expect(result[:merge_request].label_ids).to eq(labels.map(&:id)) + end + + it "can inherit milestones" do + result = service2.execute + + expect(result[:merge_request].milestone_id).to eq(milestone.id) + end + it 'delegates the branch creation to CreateBranchService' do expect_any_instance_of(CreateBranchService).to receive(:execute).once.and_call_original -- cgit v1.2.1 From 1066d8ba77ba242851c906aa523bd7548dad1d15 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 16 Aug 2017 13:30:49 +0200 Subject: Use usual method to retrieve CI/CD stage statuses --- app/models/ci/stage.rb | 4 ++-- spec/factories/ci/stages.rb | 2 +- spec/lib/gitlab/import_export/all_models.yml | 2 +- spec/models/ci/stage_spec.rb | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 7819bc3cd2c..4ee972fa68d 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -10,7 +10,7 @@ module Ci belongs_to :project belongs_to :pipeline - has_many :commit_statuses, foreign_key: :stage_id + has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id has_many :builds, foreign_key: :stage_id validates :project, presence: true, unless: :importing? @@ -50,7 +50,7 @@ module Ci def update_status retry_optimistic_lock(self) do - case commit_statuses.latest.status + case statuses.latest.status when 'pending' then enqueue when 'running' then run when 'success' then succeed diff --git a/spec/factories/ci/stages.rb b/spec/factories/ci/stages.rb index bace932cf99..b2ded945738 100644 --- a/spec/factories/ci/stages.rb +++ b/spec/factories/ci/stages.rb @@ -17,7 +17,7 @@ FactoryGirl.define do end factory :ci_stage_entity, class: Ci::Stage do - project factory: :empty_project + project factory: :project pipeline factory: :ci_empty_pipeline name 'test' diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index f323777b13a..8da02b0cf00 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -119,7 +119,7 @@ pipeline_variables: stages: - project - pipeline -- commit_statuses +- statuses - builds statuses: - project diff --git a/spec/models/ci/stage_spec.rb b/spec/models/ci/stage_spec.rb index d5c66598451..74c9d6145e2 100644 --- a/spec/models/ci/stage_spec.rb +++ b/spec/models/ci/stage_spec.rb @@ -9,9 +9,9 @@ describe Ci::Stage, :models do create(:commit_status, stage_id: stage.id) end - describe '#commit_statuses' do + describe '#statuses' do it 'returns all commit statuses' do - expect(stage.commit_statuses.count).to be 2 + expect(stage.statuses.count).to be 2 end end -- cgit v1.2.1 From 3a1103fd9173e8cb7a70c871d6a54a846f6eee4a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 17 Aug 2017 12:27:02 +0200 Subject: Add specs for stage status background migration class --- .../migrate_stage_status_spec.rb | 80 ++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 spec/lib/gitlab/background_migration/migrate_stage_status_spec.rb diff --git a/spec/lib/gitlab/background_migration/migrate_stage_status_spec.rb b/spec/lib/gitlab/background_migration/migrate_stage_status_spec.rb new file mode 100644 index 00000000000..878158910be --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_stage_status_spec.rb @@ -0,0 +1,80 @@ +require 'spec_helper' + +describe Gitlab::BackgroundMigration::MigrateStageStatus, :migration, schema: 20170711145320 do + let(:projects) { table(:projects) } + let(:pipelines) { table(:ci_pipelines) } + let(:stages) { table(:ci_stages) } + let(:jobs) { table(:ci_builds) } + + STATUSES = { created: 0, pending: 1, running: 2, success: 3, + failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze + + before do + projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1') + pipelines.create!(id: 1, project_id: 1, ref: 'master', sha: 'adf43c3a') + stages.create!(id: 1, pipeline_id: 1, project_id: 1, name: 'test', status: nil) + stages.create!(id: 2, pipeline_id: 1, project_id: 1, name: 'deploy', status: nil) + end + + context 'when stage status is known' do + before do + create_job(project: 1, pipeline: 1, stage: 'test', status: 'success') + create_job(project: 1, pipeline: 1, stage: 'test', status: 'running') + create_job(project: 1, pipeline: 1, stage: 'deploy', status: 'failed') + end + + it 'sets a correct stage status' do + described_class.new.perform(1, 2) + + expect(stages.first.status).to eq STATUSES[:running] + expect(stages.second.status).to eq STATUSES[:failed] + end + end + + context 'when stage status is not known' do + it 'sets a skipped stage status' do + described_class.new.perform(1, 2) + + expect(stages.first.status).to eq STATUSES[:skipped] + expect(stages.second.status).to eq STATUSES[:skipped] + end + end + + context 'when stage status includes status of a retried job' do + before do + create_job(project: 1, pipeline: 1, stage: 'test', status: 'canceled') + create_job(project: 1, pipeline: 1, stage: 'deploy', status: 'failed', retried: true) + create_job(project: 1, pipeline: 1, stage: 'deploy', status: 'success') + end + + it 'sets a correct stage status' do + described_class.new.perform(1, 2) + + expect(stages.first.status).to eq STATUSES[:canceled] + expect(stages.second.status).to eq STATUSES[:success] + end + end + + context 'when some job in the stage is blocked / manual' do + before do + create_job(project: 1, pipeline: 1, stage: 'test', status: 'failed') + create_job(project: 1, pipeline: 1, stage: 'test', status: 'manual') + create_job(project: 1, pipeline: 1, stage: 'deploy', status: 'success', when: 'manual') + end + + it 'sets a correct stage status' do + described_class.new.perform(1, 2) + + expect(stages.first.status).to eq STATUSES[:manual] + expect(stages.second.status).to eq STATUSES[:success] + end + end + + def create_job(project:, pipeline:, stage:, status:, **opts) + stages = { test: 1, build: 2, deploy: 3 } + + jobs.create!(project_id: project, commit_id: pipeline, + stage_idx: stages[stage.to_sym], stage: stage, + status: status, **opts) + end +end -- cgit v1.2.1 From d368b2d20230c986dbfc813312bbf25970208aaf Mon Sep 17 00:00:00 2001 From: haseeb Date: Thu, 17 Aug 2017 20:14:37 +0530 Subject: moved merge parms to service --- app/controllers/projects/issues_controller.rb | 12 +------- .../merge_requests/create_from_issue_service.rb | 5 +++- .../create_from_issue_service_spec.rb | 33 ++++++++-------------- 3 files changed, 17 insertions(+), 33 deletions(-) diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index 04c14806544..8893a514207 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -212,7 +212,7 @@ class Projects::IssuesController < Projects::ApplicationController end def create_merge_request - result = ::MergeRequests::CreateFromIssueService.new(project, current_user, merge_request_params).execute + result = ::MergeRequests::CreateFromIssueService.new(project, current_user, issue_iid: issue.iid).execute if result[:status] == :success render json: MergeRequestCreateSerializer.new.represent(result[:merge_request]) @@ -287,14 +287,4 @@ class Projects::IssuesController < Projects::ApplicationController redirect_to new_user_session_path, notice: notice end - - private - - def merge_request_params - { - issue_iid: issue.iid, - label_ids: issue.label_ids, - milestone_id: issue.milestone_id - } - end end diff --git a/app/services/merge_requests/create_from_issue_service.rb b/app/services/merge_requests/create_from_issue_service.rb index 738cedbaed7..6bc70d8175e 100644 --- a/app/services/merge_requests/create_from_issue_service.rb +++ b/app/services/merge_requests/create_from_issue_service.rb @@ -3,6 +3,8 @@ module MergeRequests def execute return error('Invalid issue iid') unless issue_iid.present? && issue.present? + params[:label_ids] = issue.label_ids if issue.label_ids.any? + result = CreateBranchService.new(project, current_user).execute(branch_name, ref) return result if result[:status] == :error @@ -43,7 +45,8 @@ module MergeRequests { source_project_id: project.id, source_branch: branch_name, - target_project_id: project.id + target_project_id: project.id, + milestone_id: issue.milestone_id } end diff --git a/spec/services/merge_requests/create_from_issue_service_spec.rb b/spec/services/merge_requests/create_from_issue_service_spec.rb index bdc53c7e64b..b3c0e6518ef 100644 --- a/spec/services/merge_requests/create_from_issue_service_spec.rb +++ b/spec/services/merge_requests/create_from_issue_service_spec.rb @@ -3,22 +3,11 @@ require 'spec_helper' describe MergeRequests::CreateFromIssueService do let(:project) { create(:project, :repository) } let(:user) { create(:user) } - let(:issue) { create(:issue, project: project) } - let(:milestone) { create(:milestone, project: project) } - let(:labels) { create_pair(:label, project: project) } - + let(:label_ids) { create_pair(:label, project: project).map(&:id) } + let(:milestone_id) { create(:milestone, project: project).id } + let(:issue) { create(:issue, project: project, milestone_id: milestone_id) } + subject(:service) { described_class.new(project, user, issue_iid: issue.iid) } - subject(:service2) do - described_class.new( - project, - user, - { - issue_iid: issue.iid, - milestone_id: milestone.id, - label_ids: labels.map(&:id) - } - ) - end before do project.add_developer(user) @@ -38,16 +27,18 @@ describe MergeRequests::CreateFromIssueService do described_class.new(project, user, issue_iid: -1).execute end - it "can inherit labels" do - result = service2.execute + it "inherits labels" do + issue.assign_attributes(label_ids: label_ids) - expect(result[:merge_request].label_ids).to eq(labels.map(&:id)) + result = service.execute + + expect(result[:merge_request].label_ids).to eq(label_ids) end - it "can inherit milestones" do - result = service2.execute + it "inherits milestones" do + result = service.execute - expect(result[:merge_request].milestone_id).to eq(milestone.id) + expect(result[:merge_request].milestone_id).to eq(milestone_id) end it 'delegates the branch creation to CreateBranchService' do -- cgit v1.2.1 From d6acd8ec2f0c60ef6c380c23b679af9df2c5a1ac Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 18 Aug 2017 01:43:17 +0900 Subject: Add validate: false. Add validates uniqueness. --- app/models/ci/pipeline_schedule.rb | 2 +- app/models/ci/pipeline_schedule_variable.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index 085eeeae157..e7e02587759 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -9,7 +9,7 @@ module Ci belongs_to :owner, class_name: 'User' has_one :last_pipeline, -> { order(id: :desc) }, class_name: 'Ci::Pipeline' has_many :pipelines - has_many :variables, class_name: 'Ci::PipelineScheduleVariable' + has_many :variables, class_name: 'Ci::PipelineScheduleVariable', validate: false validates :cron, unless: :importing?, cron: true, presence: { unless: :importing? } validates :cron_timezone, cron_timezone: true, presence: { unless: :importing? } diff --git a/app/models/ci/pipeline_schedule_variable.rb b/app/models/ci/pipeline_schedule_variable.rb index 1ff177616e8..ee5b8733fac 100644 --- a/app/models/ci/pipeline_schedule_variable.rb +++ b/app/models/ci/pipeline_schedule_variable.rb @@ -4,5 +4,7 @@ module Ci include HasVariable belongs_to :pipeline_schedule + + validates :key, uniqueness: { scope: :pipeline_schedule_id } end end -- cgit v1.2.1 From 1ffd0c8562922de0e70eef2faf71108e0eecbb51 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Fri, 18 Aug 2017 10:14:13 +0200 Subject: Remove CI API v1 This API was mainly for internal usage, and has been moved to the general API: APIv4. The endpoints have been deprecated since 9.0, and won't see 10.0. :) --- changelogs/unreleased/zj-remove-ci-api-v1.yml | 5 + config/routes/ci.rb | 4 - doc/api/ci/README.md | 24 - doc/api/ci/builds.md | 147 ----- doc/api/ci/lint.md | 51 -- doc/api/ci/runners.md | 59 -- doc/ci/api/README.md | 1 - doc/ci/api/builds.md | 1 - doc/ci/api/runners.md | 1 - doc/topics/authentication/index.md | 1 - lib/ci/api/api.rb | 39 -- lib/ci/api/builds.rb | 219 ------- lib/ci/api/entities.rb | 93 --- lib/ci/api/helpers.rb | 89 --- lib/ci/api/runners.rb | 50 -- lib/ci/api/triggers.rb | 39 -- spec/requests/ci/api/builds_spec.rb | 912 -------------------------- spec/requests/ci/api/runners_spec.rb | 127 ---- spec/requests/ci/api/triggers_spec.rb | 90 --- 19 files changed, 5 insertions(+), 1947 deletions(-) create mode 100644 changelogs/unreleased/zj-remove-ci-api-v1.yml delete mode 100644 doc/api/ci/README.md delete mode 100644 doc/api/ci/builds.md delete mode 100644 doc/api/ci/lint.md delete mode 100644 doc/api/ci/runners.md delete mode 100644 doc/ci/api/README.md delete mode 100644 doc/ci/api/builds.md delete mode 100644 doc/ci/api/runners.md delete mode 100644 lib/ci/api/api.rb delete mode 100644 lib/ci/api/builds.rb delete mode 100644 lib/ci/api/entities.rb delete mode 100644 lib/ci/api/helpers.rb delete mode 100644 lib/ci/api/runners.rb delete mode 100644 lib/ci/api/triggers.rb delete mode 100644 spec/requests/ci/api/builds_spec.rb delete mode 100644 spec/requests/ci/api/runners_spec.rb delete mode 100644 spec/requests/ci/api/triggers_spec.rb diff --git a/changelogs/unreleased/zj-remove-ci-api-v1.yml b/changelogs/unreleased/zj-remove-ci-api-v1.yml new file mode 100644 index 00000000000..8f2dc321b36 --- /dev/null +++ b/changelogs/unreleased/zj-remove-ci-api-v1.yml @@ -0,0 +1,5 @@ +--- +title: Remove CI API v1 +merge_request: +author: +type: removed diff --git a/config/routes/ci.rb b/config/routes/ci.rb index 8d23aa8fbf6..cbd4c2db852 100644 --- a/config/routes/ci.rb +++ b/config/routes/ci.rb @@ -1,8 +1,4 @@ namespace :ci do - # CI API - Ci::API::API.logger Rails.logger - mount Ci::API::API => '/api' - resource :lint, only: [:show, :create] root to: redirect('/') diff --git a/doc/api/ci/README.md b/doc/api/ci/README.md deleted file mode 100644 index 96a281e27c8..00000000000 --- a/doc/api/ci/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# GitLab CI API - -## Purpose - -The main purpose of GitLab CI API is to provide the necessary data and context -for GitLab CI Runners. - -All relevant information about the consumer API can be found in a -[separate document](../../api/README.md). - -## API Prefix - -The current CI API prefix is `/ci/api/v1`. - -You need to prepend this prefix to all examples in this documentation, like: - -```bash -GET /ci/api/v1/builds/:id/artifacts -``` - -## Resources - -- [Builds](builds.md) -- [Runners](runners.md) diff --git a/doc/api/ci/builds.md b/doc/api/ci/builds.md deleted file mode 100644 index c8374d94716..00000000000 --- a/doc/api/ci/builds.md +++ /dev/null @@ -1,147 +0,0 @@ -# Builds API - -API used by runners to receive and update builds. - ->**Note:** -This API is intended to be used only by Runners as their own -communication channel. For the consumer API see the -[Jobs API](../jobs.md). - -## Authentication - -This API uses two types of authentication: - -1. Unique Runner's token which is the token assigned to the Runner after it - has been registered. - -2. Using the build authorization token. - This is project's CI token that can be found under the **Builds** section of - a project's settings. The build authorization token can be passed as a - parameter or a value of `BUILD-TOKEN` header. - -These two methods of authentication are interchangeable. - -## Builds - -### Runs oldest pending build by runner - -``` -POST /ci/api/v1/builds/register -``` - -| Attribute | Type | Required | Description | -|-----------|---------|----------|---------------------| -| `token` | string | yes | Unique runner token | - - -``` -curl --request POST "https://gitlab.example.com/ci/api/v1/builds/register" --form "token=t0k3n" -``` - -**Responses:** - -| Status | Data |Description | -|--------|------|---------------------------------------------------------------------------| -| `201` | yes | When a build is scheduled for a runner | -| `204` | no | When no builds are scheduled for a runner (for GitLab Runner >= `v1.3.0`) | -| `403` | no | When invalid token is used or no token is sent | -| `404` | no | When no builds are scheduled for a runner (for GitLab Runner < `v1.3.0`) **or** when the runner is set to `paused` in GitLab runner's configuration page | - -### Update details of an existing build - -``` -PUT /ci/api/v1/builds/:id -``` - -| Attribute | Type | Required | Description | -|-----------|---------|----------|----------------------| -| `id` | integer | yes | The ID of a project | -| `token` | string | yes | Unique runner token | -| `state` | string | no | The state of a build | -| `trace` | string | no | The trace of a build | - -``` -curl --request PUT "https://gitlab.example.com/ci/api/v1/builds/1234" --form "token=t0k3n" --form "state=running" --form "trace=Running git clone...\n" -``` - -### Incremental build trace update - -Using this method you need to send trace content as a request body. You also need to provide the `Content-Range` header -with a range of sent trace part. Note that you need to send parts in the proper order, so the begining of the part -must start just after the end of the previous part. If you provide the wrong part, then GitLab CI API will return `416 -Range Not Satisfiable` response with a header `Range: 0-X`, where `X` is the current trace length. - -For example, if you receive `Range: 0-11` in the response, then your next part must contain a `Content-Range: 11-...` -header and a trace part covered by this range. - -For a valid update API will return `202` response with: -* `Build-Status: {status}` header containing current status of the build, -* `Range: 0-{length}` header with the current trace length. - -``` -PATCH /ci/api/v1/builds/:id/trace.txt -``` - -Parameters: - -| Attribute | Type | Required | Description | -|-----------|---------|----------|----------------------| -| `id` | integer | yes | The ID of a build | - -Headers: - -| Attribute | Type | Required | Description | -|-----------------|---------|----------|-----------------------------------| -| `BUILD-TOKEN` | string | yes | The build authorization token | -| `Content-Range` | string | yes | Bytes range of trace that is sent | - -``` -curl --request PATCH "https://gitlab.example.com/ci/api/v1/builds/1234/trace.txt" --header "BUILD-TOKEN=build_t0k3n" --header "Content-Range=0-21" --data "Running git clone...\n" -``` - - -### Upload artifacts to build - -``` -POST /ci/api/v1/builds/:id/artifacts -``` - -| Attribute | Type | Required | Description | -|-----------|---------|----------|-------------------------------| -| `id` | integer | yes | The ID of a build | -| `token` | string | yes | The build authorization token | -| `file` | mixed | yes | Artifacts file | - -``` -curl --request POST "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" --form "token=build_t0k3n" --form "file=@/path/to/file" -``` - -### Download the artifacts file from build - -``` -GET /ci/api/v1/builds/:id/artifacts -``` - -| Attribute | Type | Required | Description | -|-----------|---------|----------|-------------------------------| -| `id` | integer | yes | The ID of a build | -| `token` | string | yes | The build authorization token | - -``` -curl "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" --form "token=build_t0k3n" -``` - -### Remove the artifacts file from build - -``` -DELETE /ci/api/v1/builds/:id/artifacts -``` - -| Attribute | Type | Required | Description | -|-----------|---------|----------|-------------------------------| -| ` id` | integer | yes | The ID of a build | -| `token` | string | yes | The build authorization token | - -``` -curl --request DELETE "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" --form "token=build_t0k3n" -``` diff --git a/doc/api/ci/lint.md b/doc/api/ci/lint.md deleted file mode 100644 index e4a6dc809b1..00000000000 --- a/doc/api/ci/lint.md +++ /dev/null @@ -1,51 +0,0 @@ -# Validate the .gitlab-ci.yml (API) - -> [Introduced][ce-5953] in GitLab 8.12. - -Checks if your .gitlab-ci.yml file is valid. - -``` -POST ci/lint -``` - -| Attribute | Type | Required | Description | -| ---------- | ------- | -------- | -------- | -| `content` | string | yes | the .gitlab-ci.yaml content| - -```bash -curl --header "Content-Type: application/json" https://gitlab.example.com/api/v4/ci/lint --data '{"content": "{ \"image\": \"ruby:2.1\", \"services\": [\"postgres\"], \"before_script\": [\"gem install bundler\", \"bundle install\", \"bundle exec rake db:create\"], \"variables\": {\"DB_NAME\": \"postgres\"}, \"types\": [\"test\", \"deploy\", \"notify\"], \"rspec\": { \"script\": \"rake spec\", \"tags\": [\"ruby\", \"postgres\"], \"only\": [\"branches\"]}}"}' -``` - -Be sure to copy paste the exact contents of `.gitlab-ci.yml` as YAML is very picky about indentation and spaces. - -Example responses: - -* Valid content: - - ```json - { - "status": "valid", - "errors": [] - } - ``` - -* Invalid content: - - ```json - { - "status": "invalid", - "errors": [ - "variables config should be a hash of key value pairs" - ] - } - ``` - -* Without the content attribute: - - ```json - { - "error": "content is missing" - } - ``` - -[ce-5953]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5953 diff --git a/doc/api/ci/runners.md b/doc/api/ci/runners.md deleted file mode 100644 index 342c039dad8..00000000000 --- a/doc/api/ci/runners.md +++ /dev/null @@ -1,59 +0,0 @@ -# Register and Delete Runners API - -API used by Runners to register and delete themselves. - ->**Note:** -This API is intended to be used only by Runners as their own -communication channel. For the consumer API see the -[new Runners API](../runners.md). - -## Authentication - -This API uses two types of authentication: - -1. Unique Runner's token, which is the token assigned to the Runner after it - has been registered. This token can be found on the Runner's edit page (go to - **Project > Runners**, select one of the Runners listed under **Runners activated for - this project**). - -2. Using Runners' registration token. - This is a token that can be found in project's settings. - It can also be found in the **Admin > Runners** settings area. - There are two types of tokens you can pass: shared Runner registration - token or project specific registration token. - -## Register a new runner - -Used to make GitLab CI aware of available runners. - -```sh -POST /ci/api/v1/runners/register -``` - -| Attribute | Type | Required | Description | -| --------- | ------- | --------- | ----------- | -| `token` | string | yes | Runner's registration token | - -Example request: - -```sh -curl --request POST "https://gitlab.example.com/ci/api/v1/runners/register" --form "token=t0k3n" -``` - -## Delete a Runner - -Used to remove a Runner. - -```sh -DELETE /ci/api/v1/runners/delete -``` - -| Attribute | Type | Required | Description | -| --------- | ------- | --------- | ----------- | -| `token` | string | yes | Unique Runner's token | - -Example request: - -```sh -curl --request DELETE "https://gitlab.example.com/ci/api/v1/runners/delete" --form "token=t0k3n" -``` diff --git a/doc/ci/api/README.md b/doc/ci/api/README.md deleted file mode 100644 index 98f37935427..00000000000 --- a/doc/ci/api/README.md +++ /dev/null @@ -1 +0,0 @@ -This document was moved to a [new location](../../api/ci/README.md). diff --git a/doc/ci/api/builds.md b/doc/ci/api/builds.md deleted file mode 100644 index 0563a367609..00000000000 --- a/doc/ci/api/builds.md +++ /dev/null @@ -1 +0,0 @@ -This document was moved to a [new location](../../api/ci/builds.md). diff --git a/doc/ci/api/runners.md b/doc/ci/api/runners.md deleted file mode 100644 index 1027363851c..00000000000 --- a/doc/ci/api/runners.md +++ /dev/null @@ -1 +0,0 @@ -This document was moved to a [new location](../../api/ci/runners.md). diff --git a/doc/topics/authentication/index.md b/doc/topics/authentication/index.md index 0c0d482499a..fac91935a45 100644 --- a/doc/topics/authentication/index.md +++ b/doc/topics/authentication/index.md @@ -37,7 +37,6 @@ This page gathers all the resources for the topic **Authentication** within GitL - [Private Tokens](../../api/README.md#private-tokens) - [Impersonation tokens](../../api/README.md#impersonation-tokens) - [GitLab as an OAuth2 provider](../../api/oauth2.md#gitlab-as-an-oauth2-provider) -- [GitLab Runner API - Authentication](../../api/ci/runners.md#authentication) ## Third-party resources diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb deleted file mode 100644 index 24bb3649a76..00000000000 --- a/lib/ci/api/api.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Ci - module API - class API < Grape::API - include ::API::APIGuard - version 'v1', using: :path - - rescue_from ActiveRecord::RecordNotFound do - rack_response({ 'message' => '404 Not found' }.to_json, 404) - end - - # Retain 405 error rather than a 500 error for Grape 0.15.0+. - # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes - rescue_from Grape::Exceptions::MethodNotAllowed do |e| - error! e.message, e.status, e.headers - end - - rescue_from Grape::Exceptions::Base do |e| - error! e.message, e.status, e.headers - end - - rescue_from :all do |exception| - handle_api_exception(exception) - end - - content_type :txt, 'text/plain' - content_type :json, 'application/json' - format :json - - helpers ::SentryHelper - helpers ::Ci::API::Helpers - helpers ::API::Helpers - helpers Gitlab::CurrentSettings - - mount ::Ci::API::Builds - mount ::Ci::API::Runners - mount ::Ci::API::Triggers - end - end -end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb deleted file mode 100644 index 79058c02ce5..00000000000 --- a/lib/ci/api/builds.rb +++ /dev/null @@ -1,219 +0,0 @@ -module Ci - module API - # Builds API - class Builds < Grape::API - resource :builds do - # Runs oldest pending build by runner - Runners only - # - # Parameters: - # token (required) - The uniq token of runner - # - # Example Request: - # POST /builds/register - post "register" do - authenticate_runner! - required_attributes! [:token] - not_found! unless current_runner.active? - update_runner_info - - if current_runner.is_runner_queue_value_latest?(params[:last_update]) - header 'X-GitLab-Last-Update', params[:last_update] - Gitlab::Metrics.add_event(:build_not_found_cached) - return build_not_found! - end - - new_update = current_runner.ensure_runner_queue_value - - result = Ci::RegisterJobService.new(current_runner).execute - - if result.valid? - if result.build - Gitlab::Metrics.add_event(:build_found, - project: result.build.project.full_path) - - present result.build, with: Entities::BuildDetails - else - Gitlab::Metrics.add_event(:build_not_found) - - header 'X-GitLab-Last-Update', new_update - - build_not_found! - end - else - # We received build that is invalid due to concurrency conflict - Gitlab::Metrics.add_event(:build_invalid) - conflict! - end - end - - # Update an existing build - Runners only - # - # Parameters: - # id (required) - The ID of a project - # state (optional) - The state of a build - # trace (optional) - The trace of a build - # Example Request: - # PUT /builds/:id - put ":id" do - authenticate_runner! - build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) - validate_build!(build) - - update_runner_info - - build.trace.set(params[:trace]) if params[:trace] - - Gitlab::Metrics.add_event(:update_build, - project: build.project.full_path) - - case params[:state].to_s - when 'success' - build.success - when 'failed' - build.drop - end - end - - # Send incremental log update - Runners only - # - # Parameters: - # id (required) - The ID of a build - # Body: - # content of logs to append - # Headers: - # Content-Range (required) - range of content that was sent - # BUILD-TOKEN (required) - The build authorization token - # Example Request: - # PATCH /builds/:id/trace.txt - patch ":id/trace.txt" do - build = authenticate_build! - - error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') - content_range = request.headers['Content-Range'] - content_range = content_range.split('-') - - stream_size = build.trace.append(request.body.read, content_range[0].to_i) - if stream_size < 0 - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) - end - - status 202 - header 'Build-Status', build.status - header 'Range', "0-#{stream_size}" - end - - # Authorize artifacts uploading for build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # filesize (optional) - the size of uploaded file - # Example Request: - # POST /builds/:id/artifacts/authorize - post ":id/artifacts/authorize" do - require_gitlab_workhorse! - Gitlab::Workhorse.verify_api_request!(headers) - not_allowed! unless Gitlab.config.artifacts.enabled - build = authenticate_build! - forbidden!('build is not running') unless build.running? - - if params[:filesize] - file_size = params[:filesize].to_i - file_to_large! unless file_size < max_artifacts_size - end - - status 200 - content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE - Gitlab::Workhorse.artifact_upload_ok - end - - # Upload artifacts to build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # file (required) - Artifacts file - # expire_in (optional) - Specify when artifacts should expire (ex. 7d) - # Parameters (accelerated by GitLab Workhorse): - # file.path - path to locally stored body (generated by Workhorse) - # file.name - real filename as send in Content-Disposition - # file.type - real content type as send in Content-Type - # metadata.path - path to locally stored body (generated by Workhorse) - # metadata.name - filename (generated by Workhorse) - # Headers: - # BUILD-TOKEN (required) - The build authorization token, the same as token - # Body: - # The file content - # - # Example Request: - # POST /builds/:id/artifacts - post ":id/artifacts" do - require_gitlab_workhorse! - not_allowed! unless Gitlab.config.artifacts.enabled - build = authenticate_build! - forbidden!('Build is not running!') unless build.running? - - artifacts_upload_path = ArtifactUploader.artifacts_upload_path - artifacts = uploaded_file(:file, artifacts_upload_path) - metadata = uploaded_file(:metadata, artifacts_upload_path) - - bad_request!('Missing artifacts file!') unless artifacts - file_to_large! unless artifacts.size < max_artifacts_size - - build.artifacts_file = artifacts - build.artifacts_metadata = metadata - build.artifacts_expire_in = - params['expire_in'] || - Gitlab::CurrentSettings.current_application_settings - .default_artifacts_expire_in - - if build.save - present(build, with: Entities::BuildDetails) - else - render_validation_error!(build) - end - end - - # Download the artifacts file from build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Headers: - # BUILD-TOKEN (required) - The build authorization token, the same as token - # Example Request: - # GET /builds/:id/artifacts - get ":id/artifacts" do - build = authenticate_build! - artifacts_file = build.artifacts_file - - unless artifacts_file.exists? - not_found! - end - - unless artifacts_file.file_storage? - return redirect_to build.artifacts_file.url - end - - present_file!(artifacts_file.path, artifacts_file.filename) - end - - # Remove the artifacts file from build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Headers: - # BUILD-TOKEN (required) - The build authorization token, the same as token - # Example Request: - # DELETE /builds/:id/artifacts - delete ":id/artifacts" do - build = authenticate_build! - - status(200) - build.erase_artifacts! - end - end - end - end -end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb deleted file mode 100644 index 31f66dd5a58..00000000000 --- a/lib/ci/api/entities.rb +++ /dev/null @@ -1,93 +0,0 @@ -module Ci - module API - module Entities - class Commit < Grape::Entity - expose :id, :sha, :project_id, :created_at - expose :status, :finished_at, :duration - expose :git_commit_message, :git_author_name, :git_author_email - end - - class CommitWithBuilds < Commit - expose :builds - end - - class ArtifactFile < Grape::Entity - expose :filename, :size - end - - class BuildOptions < Grape::Entity - expose :image - expose :services - expose :artifacts - expose :cache - expose :dependencies - expose :after_script - end - - class Build < Grape::Entity - expose :id, :ref, :tag, :sha, :status - expose :name, :token, :stage - expose :project_id - expose :project_name - expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } - end - - class BuildCredentials < Grape::Entity - expose :type, :url, :username, :password - end - - class BuildDetails < Build - expose :commands - expose :repo_url - expose :before_sha - expose :allow_git_fetch - expose :token - expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? } - - expose :options do |model| - # This part ensures that output of old API is still the same after adding support - # for extended docker configuration options, used by new API - # - # I'm leaving this here, not in the model, because it should be removed at the same time - # when old API will be removed (planned for August 2017). - model.options.dup.tap do |options| - options[:image] = options[:image][:name] if options[:image].is_a?(Hash) - options[:services]&.map! do |service| - if service.is_a?(Hash) - service[:name] - else - service - end - end - end - end - - expose :timeout do |model| - model.timeout - end - - expose :variables - expose :depends_on_builds, using: Build - - expose :credentials, using: BuildCredentials - end - - class Runner < Grape::Entity - expose :id, :token - end - - class RunnerProject < Grape::Entity - expose :id, :project_id, :runner_id - end - - class WebHook < Grape::Entity - expose :id, :project_id, :url - end - - class TriggerRequest < Grape::Entity - expose :id, :variables - expose :pipeline, using: Commit, as: :commit - end - end - end -end diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb deleted file mode 100644 index a40b6ab6c9f..00000000000 --- a/lib/ci/api/helpers.rb +++ /dev/null @@ -1,89 +0,0 @@ -module Ci - module API - module Helpers - BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN".freeze - BUILD_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 10 * 60 - - def authenticate_runners! - forbidden! unless runner_registration_token_valid? - end - - def authenticate_runner! - forbidden! unless current_runner - end - - def authenticate_build! - build = Ci::Build.find_by_id(params[:id]) - - validate_build!(build) do - forbidden! unless build_token_valid?(build) - end - - build - end - - def validate_build!(build) - not_found! unless build - - yield if block_given? - - project = build.project - forbidden!('Project has been deleted!') if project.nil? || project.pending_delete? - forbidden!('Build has been erased!') if build.erased? - end - - def runner_registration_token_valid? - ActiveSupport::SecurityUtils.variable_size_secure_compare( - params[:token], - current_application_settings.runners_registration_token) - end - - def build_token_valid?(build) - token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s - - # We require to also check `runners_token` to maintain compatibility with old version of runners - token && (build.valid_token?(token) || build.project.valid_runners_token?(token)) - end - - def update_runner_info - return unless update_runner? - - current_runner.contacted_at = Time.now - current_runner.assign_attributes(get_runner_version_from_params) - current_runner.save if current_runner.changed? - end - - def update_runner? - # Use a random threshold to prevent beating DB updates. - # It generates a distribution between [40m, 80m]. - # - contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) - - current_runner.contacted_at.nil? || - (Time.now - current_runner.contacted_at) >= contacted_at_max_age - end - - def build_not_found! - if headers['User-Agent'].to_s =~ /gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? / - no_content! - else - not_found! - end - end - - def current_runner - @runner ||= Runner.find_by_token(params[:token].to_s) - end - - def get_runner_version_from_params - return unless params["info"].present? - attributes_for_keys(%w(name version revision platform architecture), params["info"]) - end - - def max_artifacts_size - current_application_settings.max_artifacts_size.megabytes.to_i - end - end - end -end diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb deleted file mode 100644 index 45aa2adccf5..00000000000 --- a/lib/ci/api/runners.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Ci - module API - class Runners < Grape::API - resource :runners do - desc 'Delete a runner' - params do - requires :token, type: String, desc: 'The unique token of the runner' - end - delete "delete" do - authenticate_runner! - - status(200) - Ci::Runner.find_by_token(params[:token]).destroy - end - - desc 'Register a new runner' do - success Entities::Runner - end - params do - requires :token, type: String, desc: 'The unique token of the runner' - optional :description, type: String, desc: 'The description of the runner' - optional :tag_list, type: Array[String], desc: 'A list of tags the runner should run for' - optional :run_untagged, type: Boolean, desc: 'Flag if the runner should execute untagged jobs' - optional :locked, type: Boolean, desc: 'Lock this runner for this specific project' - end - post "register" do - runner_params = declared(params, include_missing: false).except(:token) - - runner = - if runner_registration_token_valid? - # Create shared runner. Requires admin access - Ci::Runner.create(runner_params.merge(is_shared: true)) - elsif project = Project.find_by(runners_token: params[:token]) - # Create a specific runner for project. - project.runners.create(runner_params) - end - - return forbidden! unless runner - - if runner.id - runner.update(get_runner_version_from_params) - present runner, with: Entities::Runner - else - not_found! - end - end - end - end - end -end diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb deleted file mode 100644 index 6225203f223..00000000000 --- a/lib/ci/api/triggers.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Ci - module API - class Triggers < Grape::API - resource :projects do - desc 'Trigger a GitLab CI project build' do - success Entities::TriggerRequest - end - params do - requires :id, type: Integer, desc: 'The ID of a CI project' - requires :ref, type: String, desc: "The name of project's branch or tag" - requires :token, type: String, desc: 'The unique token of the trigger' - optional :variables, type: Hash, desc: 'Optional build variables' - end - post ":id/refs/:ref/trigger" do - project = Project.find_by(ci_id: params[:id]) - trigger = Ci::Trigger.find_by_token(params[:token]) - not_found! unless project && trigger - unauthorized! unless trigger.project == project - - # Validate variables - variables = params[:variables].to_h - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } - render_api_error!('variables needs to be a map of key-valued strings', 400) - end - - # create request and trigger builds - result = Ci::CreateTriggerRequestService.execute(project, trigger, params[:ref], variables) - pipeline = result.pipeline - - if pipeline.persisted? - present result.trigger_request, with: Entities::TriggerRequest - else - render_validation_error!(pipeline) - end - end - end - end - end -end diff --git a/spec/requests/ci/api/builds_spec.rb b/spec/requests/ci/api/builds_spec.rb deleted file mode 100644 index 7ccba4ba3ec..00000000000 --- a/spec/requests/ci/api/builds_spec.rb +++ /dev/null @@ -1,912 +0,0 @@ -require 'spec_helper' - -describe Ci::API::Builds do - let(:runner) { FactoryGirl.create(:ci_runner, tag_list: %w(mysql ruby)) } - let(:project) { FactoryGirl.create(:project, shared_runners_enabled: false) } - let(:last_update) { nil } - - describe "Builds API for runners" do - let(:pipeline) { create(:ci_pipeline_without_jobs, project: project, ref: 'master') } - - before do - project.runners << runner - end - - describe "POST /builds/register" do - let!(:build) { create(:ci_build, pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0) } - let(:user_agent) { 'gitlab-ci-multi-runner 1.5.2 (1-5-stable; go1.6.3; linux/amd64)' } - let!(:last_update) { } - let!(:new_update) { } - - before do - stub_container_registry_config(enabled: false) - end - - shared_examples 'no builds available' do - context 'when runner sends version in User-Agent' do - context 'for stable version' do - it 'gives 204 and set X-GitLab-Last-Update' do - expect(response).to have_http_status(204) - expect(response.header).to have_key('X-GitLab-Last-Update') - end - end - - context 'when last_update is up-to-date' do - let(:last_update) { runner.ensure_runner_queue_value } - - it 'gives 204 and set the same X-GitLab-Last-Update' do - expect(response).to have_http_status(204) - expect(response.header['X-GitLab-Last-Update']) - .to eq(last_update) - end - end - - context 'when last_update is outdated' do - let(:last_update) { runner.ensure_runner_queue_value } - let(:new_update) { runner.tick_runner_queue } - - it 'gives 204 and set a new X-GitLab-Last-Update' do - expect(response).to have_http_status(204) - expect(response.header['X-GitLab-Last-Update']) - .to eq(new_update) - end - end - - context 'for beta version' do - let(:user_agent) { 'gitlab-ci-multi-runner 1.6.0~beta.167.g2b2bacc (1-5-stable; go1.6.3; linux/amd64)' } - it { expect(response).to have_http_status(204) } - end - end - - context "when runner doesn't send version in User-Agent" do - let(:user_agent) { 'Go-http-client/1.1' } - it { expect(response).to have_http_status(404) } - end - - context "when runner doesn't have a User-Agent" do - let(:user_agent) { nil } - it { expect(response).to have_http_status(404) } - end - end - - context 'when an old image syntax is used' do - before do - build.update!(options: { image: 'codeclimate' }) - end - - it 'starts a build' do - register_builds info: { platform: :darwin } - - expect(response).to have_http_status(201) - expect(json_response["options"]).to eq({ "image" => "codeclimate" }) - end - end - - context 'when a new image syntax is used' do - before do - build.update!(options: { image: { name: 'codeclimate' } }) - end - - it 'starts a build' do - register_builds info: { platform: :darwin } - - expect(response).to have_http_status(201) - expect(json_response["options"]).to eq({ "image" => "codeclimate" }) - end - end - - context 'when an old service syntax is used' do - before do - build.update!(options: { services: ['mysql'] }) - end - - it 'starts a build' do - register_builds info: { platform: :darwin } - - expect(response).to have_http_status(201) - expect(json_response["options"]).to eq({ "services" => ["mysql"] }) - end - end - - context 'when a new service syntax is used' do - before do - build.update!(options: { services: [name: 'mysql'] }) - end - - it 'starts a build' do - register_builds info: { platform: :darwin } - - expect(response).to have_http_status(201) - expect(json_response["options"]).to eq({ "services" => ["mysql"] }) - end - end - - context 'when no image or service is defined' do - before do - build.update!(options: {}) - end - - it 'starts a build' do - register_builds info: { platform: :darwin } - - expect(response).to have_http_status(201) - - expect(json_response["options"]).to be_empty - end - end - - context 'when there is a pending build' do - it 'starts a build' do - register_builds info: { platform: :darwin } - - expect(response).to have_http_status(201) - expect(response.headers).not_to have_key('X-GitLab-Last-Update') - expect(json_response['sha']).to eq(build.sha) - expect(runner.reload.platform).to eq("darwin") - expect(json_response["options"]).to eq({ "image" => "ruby:2.1", "services" => ["postgres"] }) - expect(json_response["variables"]).to include( - { "key" => "CI_JOB_NAME", "value" => "spinach", "public" => true }, - { "key" => "CI_JOB_STAGE", "value" => "test", "public" => true }, - { "key" => "DB_NAME", "value" => "postgres", "public" => true } - ) - end - - it 'updates runner info' do - expect { register_builds }.to change { runner.reload.contacted_at } - end - - context 'when concurrently updating build' do - before do - expect_any_instance_of(Ci::Build).to receive(:run!) - .and_raise(ActiveRecord::StaleObjectError.new(nil, nil)) - end - - it 'returns a conflict' do - register_builds info: { platform: :darwin } - - expect(response).to have_http_status(409) - expect(response.headers).not_to have_key('X-GitLab-Last-Update') - end - end - - context 'registry credentials' do - let(:registry_credentials) do - { 'type' => 'registry', - 'url' => 'registry.example.com:5005', - 'username' => 'gitlab-ci-token', - 'password' => build.token } - end - - context 'when registry is enabled' do - before do - stub_container_registry_config(enabled: true, host_port: 'registry.example.com:5005') - end - - it 'sends registry credentials key' do - register_builds info: { platform: :darwin } - - expect(json_response).to have_key('credentials') - expect(json_response['credentials']).to include(registry_credentials) - end - end - - context 'when registry is disabled' do - before do - stub_container_registry_config(enabled: false, host_port: 'registry.example.com:5005') - end - - it 'does not send registry credentials' do - register_builds info: { platform: :darwin } - - expect(json_response).to have_key('credentials') - expect(json_response['credentials']).not_to include(registry_credentials) - end - end - end - - context 'when docker configuration options are used' do - let!(:build) { create(:ci_build, :extended_options, pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0) } - - it 'starts a build' do - register_builds info: { platform: :darwin } - - expect(response).to have_http_status(201) - expect(json_response['options']['image']).to eq('ruby:2.1') - expect(json_response['options']['services']).to eq(['postgres', 'docker:dind']) - end - end - end - - context 'when builds are finished' do - before do - build.success - register_builds - end - - it_behaves_like 'no builds available' - end - - context 'for other project with builds' do - before do - build.success - create(:ci_build, :pending) - register_builds - end - - it_behaves_like 'no builds available' - end - - context 'for shared runner' do - let!(:runner) { create(:ci_runner, :shared, token: "SharedRunner") } - - before do - register_builds(runner.token) - end - - it_behaves_like 'no builds available' - end - - context 'for triggered build' do - before do - trigger = create(:ci_trigger, project: project) - create(:ci_trigger_request_with_variables, pipeline: pipeline, builds: [build], trigger: trigger) - project.variables << Ci::Variable.new(key: "SECRET_KEY", value: "secret_value") - end - - it "returns variables for triggers" do - register_builds info: { platform: :darwin } - - expect(response).to have_http_status(201) - expect(json_response["variables"]).to include( - { "key" => "CI_JOB_NAME", "value" => "spinach", "public" => true }, - { "key" => "CI_JOB_STAGE", "value" => "test", "public" => true }, - { "key" => "CI_PIPELINE_TRIGGERED", "value" => "true", "public" => true }, - { "key" => "DB_NAME", "value" => "postgres", "public" => true }, - { "key" => "SECRET_KEY", "value" => "secret_value", "public" => false }, - { "key" => "TRIGGER_KEY_1", "value" => "TRIGGER_VALUE_1", "public" => false } - ) - end - end - - context 'with multiple builds' do - before do - build.success - end - - let!(:test_build) { create(:ci_build, pipeline: pipeline, name: 'deploy', stage: 'deploy', stage_idx: 1) } - - it "returns dependent builds" do - register_builds info: { platform: :darwin } - - expect(response).to have_http_status(201) - expect(json_response["id"]).to eq(test_build.id) - expect(json_response["depends_on_builds"].count).to eq(1) - expect(json_response["depends_on_builds"][0]).to include('id' => build.id, 'name' => 'spinach') - end - end - - %w(name version revision platform architecture).each do |param| - context "updates runner #{param}" do - let(:value) { "#{param}_value" } - - subject { runner.read_attribute(param.to_sym) } - - it do - register_builds info: { param => value } - - expect(response).to have_http_status(201) - runner.reload - is_expected.to eq(value) - end - end - end - - context 'when build has no tags' do - before do - build.update(tags: []) - end - - context 'when runner is allowed to pick untagged builds' do - before do - runner.update_column(:run_untagged, true) - end - - it 'picks build' do - register_builds - - expect(response).to have_http_status 201 - end - end - - context 'when runner is not allowed to pick untagged builds' do - before do - runner.update_column(:run_untagged, false) - register_builds - end - - it_behaves_like 'no builds available' - end - end - - context 'when runner is paused' do - let(:runner) { create(:ci_runner, :inactive, token: 'InactiveRunner') } - - it 'responds with 404' do - register_builds - - expect(response).to have_http_status 404 - end - - it 'does not update runner info' do - expect { register_builds } - .not_to change { runner.reload.contacted_at } - end - end - - def register_builds(token = runner.token, **params) - new_params = params.merge(token: token, last_update: last_update) - - post ci_api("/builds/register"), new_params, { 'User-Agent' => user_agent } - end - end - - describe "PUT /builds/:id" do - let(:build) { create(:ci_build, :pending, :trace, pipeline: pipeline, runner_id: runner.id) } - - before do - build.run! - put ci_api("/builds/#{build.id}"), token: runner.token - end - - it "updates a running build" do - expect(response).to have_http_status(200) - end - - it 'does not override trace information when no trace is given' do - expect(build.reload.trace.raw).to eq 'BUILD TRACE' - end - - context 'job has been erased' do - let(:build) { create(:ci_build, runner_id: runner.id, erased_at: Time.now) } - - it 'responds with forbidden' do - expect(response.status).to eq 403 - end - end - end - - describe 'PATCH /builds/:id/trace.txt' do - let(:build) do - attributes = { runner_id: runner.id, pipeline: pipeline } - create(:ci_build, :running, :trace, attributes) - end - - let(:headers) { { Ci::API::Helpers::BUILD_TOKEN_HEADER => build.token, 'Content-Type' => 'text/plain' } } - let(:headers_with_range) { headers.merge({ 'Content-Range' => '11-20' }) } - let(:update_interval) { 10.seconds.to_i } - - def patch_the_trace(content = ' appended', request_headers = nil) - unless request_headers - build.trace.read do |stream| - offset = stream.size - limit = offset + content.length - 1 - request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" }) - end - end - - Timecop.travel(build.updated_at + update_interval) do - patch ci_api("/builds/#{build.id}/trace.txt"), content, request_headers - build.reload - end - end - - def initial_patch_the_trace - patch_the_trace(' appended', headers_with_range) - end - - def force_patch_the_trace - 2.times { patch_the_trace('') } - end - - before do - initial_patch_the_trace - end - - context 'when request is valid' do - it 'gets correct response' do - expect(response.status).to eq 202 - expect(build.reload.trace.raw).to eq 'BUILD TRACE appended' - expect(response.header).to have_key 'Range' - expect(response.header).to have_key 'Build-Status' - end - - context 'when build has been updated recently' do - it { expect { patch_the_trace }.not_to change { build.updated_at }} - - it 'changes the build trace' do - patch_the_trace - - expect(build.reload.trace.raw).to eq 'BUILD TRACE appended appended' - end - - context 'when Runner makes a force-patch' do - it { expect { force_patch_the_trace }.not_to change { build.updated_at }} - - it "doesn't change the build.trace" do - force_patch_the_trace - - expect(build.reload.trace.raw).to eq 'BUILD TRACE appended' - end - end - end - - context 'when build was not updated recently' do - let(:update_interval) { 15.minutes.to_i } - - it { expect { patch_the_trace }.to change { build.updated_at } } - - it 'changes the build.trace' do - patch_the_trace - - expect(build.reload.trace.raw).to eq 'BUILD TRACE appended appended' - end - - context 'when Runner makes a force-patch' do - it { expect { force_patch_the_trace }.to change { build.updated_at } } - - it "doesn't change the build.trace" do - force_patch_the_trace - - expect(build.reload.trace.raw).to eq 'BUILD TRACE appended' - end - end - end - - context 'when project for the build has been deleted' do - let(:build) do - attributes = { runner_id: runner.id, pipeline: pipeline } - create(:ci_build, :running, :trace, attributes) do |build| - build.project.update(pending_delete: true) - end - end - - it 'responds with forbidden' do - expect(response.status).to eq(403) - end - end - end - - context 'when Runner makes a force-patch' do - before do - force_patch_the_trace - end - - it 'gets correct response' do - expect(response.status).to eq 202 - expect(build.reload.trace.raw).to eq 'BUILD TRACE appended' - expect(response.header).to have_key 'Range' - expect(response.header).to have_key 'Build-Status' - end - end - - context 'when content-range start is too big' do - let(:headers_with_range) { headers.merge({ 'Content-Range' => '15-20' }) } - - it 'gets 416 error response with range headers' do - expect(response.status).to eq 416 - expect(response.header).to have_key 'Range' - expect(response.header['Range']).to eq '0-11' - end - end - - context 'when content-range start is too small' do - let(:headers_with_range) { headers.merge({ 'Content-Range' => '8-20' }) } - - it 'gets 416 error response with range headers' do - expect(response.status).to eq 416 - expect(response.header).to have_key 'Range' - expect(response.header['Range']).to eq '0-11' - end - end - - context 'when Content-Range header is missing' do - let(:headers_with_range) { headers } - - it { expect(response.status).to eq 400 } - end - - context 'when build has been errased' do - let(:build) { create(:ci_build, runner_id: runner.id, erased_at: Time.now) } - - it { expect(response.status).to eq 403 } - end - end - - context "Artifacts" do - let(:file_upload) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') } - let(:file_upload2) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/gif') } - let(:build) { create(:ci_build, :pending, pipeline: pipeline, runner_id: runner.id) } - let(:authorize_url) { ci_api("/builds/#{build.id}/artifacts/authorize") } - let(:post_url) { ci_api("/builds/#{build.id}/artifacts") } - let(:delete_url) { ci_api("/builds/#{build.id}/artifacts") } - let(:get_url) { ci_api("/builds/#{build.id}/artifacts") } - let(:jwt_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') } - let(:headers) { { "GitLab-Workhorse" => "1.0", Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => jwt_token } } - let(:token) { build.token } - let(:headers_with_token) { headers.merge(Ci::API::Helpers::BUILD_TOKEN_HEADER => token) } - - before do - build.run! - end - - describe "POST /builds/:id/artifacts/authorize" do - context "authorizes posting artifact to running build" do - it "using token as parameter" do - post authorize_url, { token: build.token }, headers - - expect(response).to have_http_status(200) - expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) - expect(json_response["TempPath"]).not_to be_nil - end - - it "using token as header" do - post authorize_url, {}, headers_with_token - - expect(response).to have_http_status(200) - expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) - expect(json_response["TempPath"]).not_to be_nil - end - - it "using runners token" do - post authorize_url, { token: build.project.runners_token }, headers - - expect(response).to have_http_status(200) - expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) - expect(json_response["TempPath"]).not_to be_nil - end - - it "reject requests that did not go through gitlab-workhorse" do - headers.delete(Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER) - - post authorize_url, { token: build.token }, headers - - expect(response).to have_http_status(500) - end - end - - context "fails to post too large artifact" do - it "using token as parameter" do - stub_application_setting(max_artifacts_size: 0) - - post authorize_url, { token: build.token, filesize: 100 }, headers - - expect(response).to have_http_status(413) - end - - it "using token as header" do - stub_application_setting(max_artifacts_size: 0) - - post authorize_url, { filesize: 100 }, headers_with_token - - expect(response).to have_http_status(413) - end - end - - context 'authorization token is invalid' do - before do - post authorize_url, { token: 'invalid', filesize: 100 } - end - - it 'responds with forbidden' do - expect(response).to have_http_status(403) - end - end - end - - describe "POST /builds/:id/artifacts" do - context "disable sanitizer" do - before do - # by configuring this path we allow to pass temp file from any path - allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return('/') - end - - describe 'build has been erased' do - let(:build) { create(:ci_build, erased_at: Time.now) } - - before do - upload_artifacts(file_upload, headers_with_token) - end - - it 'responds with forbidden' do - expect(response.status).to eq 403 - end - end - - describe 'uploading artifacts for a running build' do - shared_examples 'successful artifacts upload' do - it 'updates successfully' do - response_filename = - json_response['artifacts_file']['filename'] - - expect(response).to have_http_status(201) - expect(response_filename).to eq(file_upload.original_filename) - end - end - - context 'uses regular file post' do - before do - upload_artifacts(file_upload, headers_with_token, false) - end - - it_behaves_like 'successful artifacts upload' - end - - context 'uses accelerated file post' do - before do - upload_artifacts(file_upload, headers_with_token, true) - end - - it_behaves_like 'successful artifacts upload' - end - - context 'updates artifact' do - before do - upload_artifacts(file_upload2, headers_with_token) - upload_artifacts(file_upload, headers_with_token) - end - - it_behaves_like 'successful artifacts upload' - end - - context 'when using runners token' do - let(:token) { build.project.runners_token } - - before do - upload_artifacts(file_upload, headers_with_token) - end - - it_behaves_like 'successful artifacts upload' - end - end - - context 'posts artifacts file and metadata file' do - let!(:artifacts) { file_upload } - let!(:metadata) { file_upload2 } - - let(:stored_artifacts_file) { build.reload.artifacts_file.file } - let(:stored_metadata_file) { build.reload.artifacts_metadata.file } - let(:stored_artifacts_size) { build.reload.artifacts_size } - - before do - post(post_url, post_data, headers_with_token) - end - - context 'posts data accelerated by workhorse is correct' do - let(:post_data) do - { 'file.path' => artifacts.path, - 'file.name' => artifacts.original_filename, - 'metadata.path' => metadata.path, - 'metadata.name' => metadata.original_filename } - end - - it 'stores artifacts and artifacts metadata' do - expect(response).to have_http_status(201) - expect(stored_artifacts_file.original_filename).to eq(artifacts.original_filename) - expect(stored_metadata_file.original_filename).to eq(metadata.original_filename) - expect(stored_artifacts_size).to eq(71759) - end - end - - context 'no artifacts file in post data' do - let(:post_data) do - { 'metadata' => metadata } - end - - it 'is expected to respond with bad request' do - expect(response).to have_http_status(400) - end - - it 'does not store metadata' do - expect(stored_metadata_file).to be_nil - end - end - end - - context 'with an expire date' do - let!(:artifacts) { file_upload } - let(:default_artifacts_expire_in) {} - - let(:post_data) do - { 'file.path' => artifacts.path, - 'file.name' => artifacts.original_filename, - 'expire_in' => expire_in } - end - - before do - stub_application_setting( - default_artifacts_expire_in: default_artifacts_expire_in) - - post(post_url, post_data, headers_with_token) - end - - context 'with an expire_in given' do - let(:expire_in) { '7 days' } - - it 'updates when specified' do - build.reload - expect(response).to have_http_status(201) - expect(json_response['artifacts_expire_at']).not_to be_empty - expect(build.artifacts_expire_at) - .to be_within(5.minutes).of(7.days.from_now) - end - end - - context 'with no expire_in given' do - let(:expire_in) { nil } - - it 'ignores if not specified' do - build.reload - expect(response).to have_http_status(201) - expect(json_response['artifacts_expire_at']).to be_nil - expect(build.artifacts_expire_at).to be_nil - end - - context 'with application default' do - context 'default to 5 days' do - let(:default_artifacts_expire_in) { '5 days' } - - it 'sets to application default' do - build.reload - expect(response).to have_http_status(201) - expect(json_response['artifacts_expire_at']) - .not_to be_empty - expect(build.artifacts_expire_at) - .to be_within(5.minutes).of(5.days.from_now) - end - end - - context 'default to 0' do - let(:default_artifacts_expire_in) { '0' } - - it 'does not set expire_in' do - build.reload - expect(response).to have_http_status(201) - expect(json_response['artifacts_expire_at']).to be_nil - expect(build.artifacts_expire_at).to be_nil - end - end - end - end - end - - context "artifacts file is too large" do - it "fails to post too large artifact" do - stub_application_setting(max_artifacts_size: 0) - upload_artifacts(file_upload, headers_with_token) - expect(response).to have_http_status(413) - end - end - - context "artifacts post request does not contain file" do - it "fails to post artifacts without file" do - post post_url, {}, headers_with_token - expect(response).to have_http_status(400) - end - end - - context 'GitLab Workhorse is not configured' do - it "fails to post artifacts without GitLab-Workhorse" do - post post_url, { token: build.token }, {} - expect(response).to have_http_status(403) - end - end - end - - context "artifacts are being stored outside of tmp path" do - before do - # by configuring this path we allow to pass file from @tmpdir only - # but all temporary files are stored in system tmp directory - @tmpdir = Dir.mktmpdir - allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return(@tmpdir) - end - - after do - FileUtils.remove_entry @tmpdir - end - - it "fails to post artifacts for outside of tmp path" do - upload_artifacts(file_upload, headers_with_token) - expect(response).to have_http_status(400) - end - end - - def upload_artifacts(file, headers = {}, accelerated = true) - if accelerated - post post_url, { - 'file.path' => file.path, - 'file.name' => file.original_filename - }, headers - else - post post_url, { file: file }, headers - end - end - end - - describe 'DELETE /builds/:id/artifacts' do - let(:build) { create(:ci_build, :artifacts) } - - before do - delete delete_url, token: build.token - end - - shared_examples 'having removable artifacts' do - it 'removes build artifacts' do - build.reload - - expect(response).to have_http_status(200) - expect(build.artifacts_file.exists?).to be_falsy - expect(build.artifacts_metadata.exists?).to be_falsy - expect(build.artifacts_size).to be_nil - end - end - - context 'when using build token' do - before do - delete delete_url, token: build.token - end - - it_behaves_like 'having removable artifacts' - end - - context 'when using runnners token' do - before do - delete delete_url, token: build.project.runners_token - end - - it_behaves_like 'having removable artifacts' - end - end - - describe 'GET /builds/:id/artifacts' do - before do - get get_url, token: token - end - - context 'build has artifacts' do - let(:build) { create(:ci_build, :artifacts) } - let(:download_headers) do - { 'Content-Transfer-Encoding' => 'binary', - 'Content-Disposition' => 'attachment; filename=ci_build_artifacts.zip' } - end - - shared_examples 'having downloadable artifacts' do - it 'download artifacts' do - expect(response).to have_http_status(200) - expect(response.headers).to include download_headers - end - end - - context 'when using build token' do - let(:token) { build.token } - - it_behaves_like 'having downloadable artifacts' - end - - context 'when using runnners token' do - let(:token) { build.project.runners_token } - - it_behaves_like 'having downloadable artifacts' - end - end - - context 'build does not has artifacts' do - let(:token) { build.token } - - it 'responds with not found' do - expect(response).to have_http_status(404) - end - end - end - end - end -end diff --git a/spec/requests/ci/api/runners_spec.rb b/spec/requests/ci/api/runners_spec.rb deleted file mode 100644 index 75059dd20a0..00000000000 --- a/spec/requests/ci/api/runners_spec.rb +++ /dev/null @@ -1,127 +0,0 @@ -require 'spec_helper' - -describe Ci::API::Runners do - include StubGitlabCalls - - let(:registration_token) { 'abcdefg123456' } - - before do - stub_gitlab_calls - stub_application_setting(runners_registration_token: registration_token) - end - - describe "POST /runners/register" do - context 'when runner token is provided' do - before do - post ci_api("/runners/register"), token: registration_token - end - - it 'creates runner with default values' do - expect(response).to have_http_status 201 - expect(Ci::Runner.first.run_untagged).to be true - expect(Ci::Runner.first.token).not_to eq(registration_token) - end - end - - context 'when runner description is provided' do - before do - post ci_api("/runners/register"), token: registration_token, - description: "server.hostname" - end - - it 'creates runner' do - expect(response).to have_http_status 201 - expect(Ci::Runner.first.description).to eq("server.hostname") - end - end - - context 'when runner tags are provided' do - before do - post ci_api("/runners/register"), token: registration_token, - tag_list: "tag1, tag2" - end - - it 'creates runner' do - expect(response).to have_http_status 201 - expect(Ci::Runner.first.tag_list.sort).to eq(%w(tag1 tag2)) - end - end - - context 'when option for running untagged jobs is provided' do - context 'when tags are provided' do - it 'creates runner' do - post ci_api("/runners/register"), token: registration_token, - run_untagged: false, - tag_list: ['tag'] - - expect(response).to have_http_status 201 - expect(Ci::Runner.first.run_untagged).to be false - end - end - - context 'when tags are not provided' do - it 'does not create runner' do - post ci_api("/runners/register"), token: registration_token, - run_untagged: false - - expect(response).to have_http_status 404 - end - end - end - - context 'when project token is provided' do - let(:project) { FactoryGirl.create(:project) } - - before do - post ci_api("/runners/register"), token: project.runners_token - end - - it 'creates runner' do - expect(response).to have_http_status 201 - expect(project.runners.size).to eq(1) - expect(Ci::Runner.first.token).not_to eq(registration_token) - expect(Ci::Runner.first.token).not_to eq(project.runners_token) - end - end - - context 'when token is invalid' do - it 'returns 403 error' do - post ci_api("/runners/register"), token: 'invalid' - - expect(response).to have_http_status 403 - end - end - - context 'when no token provided' do - it 'returns 400 error' do - post ci_api("/runners/register") - - expect(response).to have_http_status 400 - end - end - - %w(name version revision platform architecture).each do |param| - context "creates runner with #{param} saved" do - let(:value) { "#{param}_value" } - - subject { Ci::Runner.first.read_attribute(param.to_sym) } - - it do - post ci_api("/runners/register"), token: registration_token, info: { param => value } - expect(response).to have_http_status 201 - is_expected.to eq(value) - end - end - end - end - - describe "DELETE /runners/delete" do - it 'returns 200' do - runner = FactoryGirl.create(:ci_runner) - delete ci_api("/runners/delete"), token: runner.token - - expect(response).to have_http_status 200 - expect(Ci::Runner.count).to eq(0) - end - end -end diff --git a/spec/requests/ci/api/triggers_spec.rb b/spec/requests/ci/api/triggers_spec.rb deleted file mode 100644 index 7c77ebb69a2..00000000000 --- a/spec/requests/ci/api/triggers_spec.rb +++ /dev/null @@ -1,90 +0,0 @@ -require 'spec_helper' - -describe Ci::API::Triggers do - describe 'POST /projects/:project_id/refs/:ref/trigger' do - let!(:trigger_token) { 'secure token' } - let!(:project) { create(:project, :repository, ci_id: 10) } - let!(:project2) { create(:project, ci_id: 11) } - - let!(:trigger) do - create(:ci_trigger, - project: project, - token: trigger_token, - owner: create(:user)) - end - - let(:options) do - { - token: trigger_token - } - end - - before do - stub_ci_pipeline_to_return_yaml_file - - project.add_developer(trigger.owner) - end - - context 'Handles errors' do - it 'returns bad request if token is missing' do - post ci_api("/projects/#{project.ci_id}/refs/master/trigger") - expect(response).to have_http_status(400) - end - - it 'returns not found if project is not found' do - post ci_api('/projects/0/refs/master/trigger'), options - expect(response).to have_http_status(404) - end - - it 'returns unauthorized if token is for different project' do - post ci_api("/projects/#{project2.ci_id}/refs/master/trigger"), options - expect(response).to have_http_status(401) - end - end - - context 'Have a commit' do - let(:pipeline) { project.pipelines.last } - - it 'creates builds' do - post ci_api("/projects/#{project.ci_id}/refs/master/trigger"), options - expect(response).to have_http_status(201) - pipeline.builds.reload - expect(pipeline.builds.pending.size).to eq(2) - expect(pipeline.builds.size).to eq(5) - end - - it 'returns bad request with no builds created if there\'s no commit for that ref' do - post ci_api("/projects/#{project.ci_id}/refs/other-branch/trigger"), options - expect(response).to have_http_status(400) - expect(json_response['message']['base']) - .to contain_exactly('Reference not found') - end - - context 'Validates variables' do - let(:variables) do - { 'TRIGGER_KEY' => 'TRIGGER_VALUE' } - end - - it 'validates variables to be a hash' do - post ci_api("/projects/#{project.ci_id}/refs/master/trigger"), options.merge(variables: 'value') - expect(response).to have_http_status(400) - - expect(json_response['error']).to eq('variables is invalid') - end - - it 'validates variables needs to be a map of key-valued strings' do - post ci_api("/projects/#{project.ci_id}/refs/master/trigger"), options.merge(variables: { key: %w(1 2) }) - expect(response).to have_http_status(400) - expect(json_response['message']).to eq('variables needs to be a map of key-valued strings') - end - - it 'creates trigger request with variables' do - post ci_api("/projects/#{project.ci_id}/refs/master/trigger"), options.merge(variables: variables) - expect(response).to have_http_status(201) - pipeline.builds.reload - expect(pipeline.builds.first.trigger_request.variables).to eq(variables) - end - end - end - end -end -- cgit v1.2.1 From accaf7eaf268b340b39b6ef9b949cc78988b0e33 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 18 Aug 2017 13:49:02 +0200 Subject: Make create pipeline service specs more readable --- spec/services/ci/create_pipeline_service_spec.rb | 38 +++++++++++++++++------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index 53d4fcfed18..bf68ee0e64d 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -55,10 +55,15 @@ describe Ci::CreatePipelineService do context 'when merge requests already exist for this source branch' do it 'updates head pipeline of each merge request' do - merge_request_1 = create(:merge_request, source_branch: 'master', target_branch: "branch_1", source_project: project) - merge_request_2 = create(:merge_request, source_branch: 'master', target_branch: "branch_2", source_project: project) + merge_request_1 = create(:merge_request, source_branch: 'master', + target_branch: "branch_1", + source_project: project) - head_pipeline = pipeline + merge_request_2 = create(:merge_request, source_branch: 'master', + target_branch: "branch_2", + source_project: project) + + head_pipeline = execute_service expect(merge_request_1.reload.head_pipeline).to eq(head_pipeline) expect(merge_request_2.reload.head_pipeline).to eq(head_pipeline) @@ -66,9 +71,11 @@ describe Ci::CreatePipelineService do context 'when there is no pipeline for source branch' do it "does not update merge request head pipeline" do - merge_request = create(:merge_request, source_branch: 'feature', target_branch: "branch_1", source_project: project) + merge_request = create(:merge_request, source_branch: 'feature', + target_branch: "branch_1", + source_project: project) - head_pipeline = pipeline + head_pipeline = execute_service expect(merge_request.reload.head_pipeline).not_to eq(head_pipeline) end @@ -76,13 +83,19 @@ describe Ci::CreatePipelineService do context 'when merge request target project is different from source project' do let!(:target_project) { create(:project, :repository) } - let!(:forked_project_link) { create(:forked_project_link, forked_to_project: project, forked_from_project: target_project) } + + let!(:forked_project_link) do + create(:forked_project_link, forked_to_project: project, + forked_from_project: target_project) + end it 'updates head pipeline for merge request' do - merge_request = - create(:merge_request, source_branch: 'master', target_branch: "branch_1", source_project: project, target_project: target_project) + merge_request = create(:merge_request, source_branch: 'master', + target_branch: "branch_1", + source_project: project, + target_project: target_project) - head_pipeline = pipeline + head_pipeline = execute_service expect(merge_request.reload.head_pipeline).to eq(head_pipeline) end @@ -90,9 +103,12 @@ describe Ci::CreatePipelineService do context 'when the pipeline is not the latest for the branch' do it 'does not update merge request head pipeline' do - merge_request = create(:merge_request, source_branch: 'master', target_branch: "branch_1", source_project: project) + merge_request = create(:merge_request, source_branch: 'master', + target_branch: "branch_1", + source_project: project) - allow_any_instance_of(Ci::Pipeline).to receive(:latest?).and_return(false) + allow_any_instance_of(Ci::Pipeline) + .to receive(:latest?).and_return(false) pipeline -- cgit v1.2.1 From 66afd1ee1cfdefbcc971ab438081e85b6dde044b Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 17 Aug 2017 18:34:26 +0200 Subject: Add Gitaly annotation --- lib/gitlab/git/repository.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 53aa5b12489..246bfd57787 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -126,6 +126,8 @@ module Gitlab # This is to work around a bug in libgit2 that causes in-memory refs to # be stale/invalid when packed-refs is changed. # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333 + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/474 def find_branch(name, force_reload = false) reload_rugged if force_reload -- cgit v1.2.1 From faaed0aa1f6b26f6b8f69f1c59ae7faa8ccf9ce5 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 17 Aug 2017 18:34:43 +0200 Subject: Delete unused Gitlab::Git methods --- lib/gitlab/git/repository.rb | 20 -------------------- spec/lib/gitlab/git/repository_spec.rb | 17 ----------------- 2 files changed, 37 deletions(-) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 246bfd57787..5152d212a69 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -64,7 +64,6 @@ module Gitlab end delegate :empty?, - :bare?, to: :rugged delegate :exists?, to: :gitaly_repository_client @@ -233,10 +232,6 @@ module Gitlab branch_names + tag_names end - def has_commits? - !empty? - end - # Discovers the default branch based on the repository's available branches # # - If no branches are present, returns nil @@ -594,11 +589,6 @@ module Gitlab raise InvalidRef.new("Invalid reference #{start_point}") end - # Return an array of this repository's remote names - def remote_names - rugged.remotes.each_name.to_a - end - # Delete the specified remote from this repository. def remote_delete(remote_name) rugged.remotes.delete(remote_name) @@ -618,16 +608,6 @@ module Gitlab rugged.remotes.set_url(remote_name, options[:url]) if options[:url] end - # Fetch the specified remote - def fetch(remote_name) - rugged.remotes[remote_name].fetch - end - - # Push +*refspecs+ to the remote identified by +remote_name+. - def push(remote_name, *refspecs) - rugged.remotes[remote_name].push(refspecs) - end - AUTOCRLF_VALUES = { "true" => true, "false" => false, diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index a3ae0a4686d..8ec8dfe6acf 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -235,18 +235,10 @@ describe Gitlab::Git::Repository, seed_helper: true do it { is_expected.to be < 2 } end - describe '#has_commits?' do - it { expect(repository.has_commits?).to be_truthy } - end - describe '#empty?' do it { expect(repository.empty?).to be_falsey } end - describe '#bare?' do - it { expect(repository.bare?).to be_truthy } - end - describe '#ref_names' do let(:ref_names) { repository.ref_names } subject { ref_names } @@ -441,15 +433,6 @@ describe Gitlab::Git::Repository, seed_helper: true do end end - describe "#remote_names" do - let(:remotes) { repository.remote_names } - - it "should have one entry: 'origin'" do - expect(remotes.size).to eq(1) - expect(remotes.first).to eq("origin") - end - end - describe "#refs_hash" do let(:refs) { repository.refs_hash } -- cgit v1.2.1 From 22f1b04637a22c44c4f3ae591d180f5335d3ae16 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 18 Aug 2017 14:12:40 +0200 Subject: Update merge request pipeline even if if has errors --- app/services/ci/create_pipeline_service.rb | 11 ++++++++--- spec/services/ci/create_pipeline_service_spec.rb | 20 +++++++++++++++++++- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index 884b681ff81..d0ba9f89460 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -176,9 +176,14 @@ module Ci end def error(message, save: false) - pipeline.errors.add(:base, message) - pipeline.drop if save - pipeline + pipeline.tap do + pipeline.errors.add(:base, message) + + if save + pipeline.drop + update_merge_requests_head_pipeline + end + end end def pipeline_created_counter diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index bf68ee0e64d..8465a6f99bd 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -110,11 +110,29 @@ describe Ci::CreatePipelineService do allow_any_instance_of(Ci::Pipeline) .to receive(:latest?).and_return(false) - pipeline + execute_service expect(merge_request.reload.head_pipeline).to be_nil end end + + context 'when pipeline has errors' do + before do + stub_ci_pipeline_yaml_file('some invalid syntax') + end + + it 'updates merge request head pipeline reference' do + merge_request = create(:merge_request, source_branch: 'master', + target_branch: 'feature', + source_project: project) + + head_pipeline = execute_service + + expect(head_pipeline).to be_persisted + expect(head_pipeline.yaml_errors).to be_present + expect(merge_request.reload.head_pipeline).to eq head_pipeline + end + end end context 'auto-cancel enabled' do -- cgit v1.2.1 From a902166475772521201196f3f9984c663c6c2ee5 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 18 Aug 2017 15:55:30 +0200 Subject: More annotations --- lib/gitlab/git/repository.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 5152d212a69..163ffb6efc5 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -571,6 +571,8 @@ module Gitlab end # Delete the specified branch from the repository + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476 def delete_branch(branch_name) rugged.branches.delete(branch_name) end @@ -580,6 +582,8 @@ module Gitlab # Examples: # create_branch("feature") # create_branch("other-feature", "master") + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476 def create_branch(ref, start_point = "HEAD") rugged_ref = rugged.branches.create(ref, start_point) target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) -- cgit v1.2.1 From df7f2b135a1d811761272ef0f9140ae5aa16c01d Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 18 Aug 2017 15:59:17 +0200 Subject: Simplify method arguments --- lib/gitlab/git/repository.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 163ffb6efc5..39263d9e6fc 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -607,9 +607,9 @@ module Gitlab # # Example # repo.update_remote("origin", url: "path/to/repo") - def remote_update(remote_name, options = {}) + def remote_update(remote_name, url:) # TODO: Implement other remote options - rugged.remotes.set_url(remote_name, options[:url]) if options[:url] + rugged.remotes.set_url(remote_name, url) end AUTOCRLF_VALUES = { -- cgit v1.2.1 From 3049dfaf48676ecdf53d0d07297432fbaf4ca720 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 18 Aug 2017 16:03:27 +0200 Subject: Simplify return values --- lib/gitlab/git/repository.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 39263d9e6fc..eb3731ba35a 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -596,11 +596,13 @@ module Gitlab # Delete the specified remote from this repository. def remote_delete(remote_name) rugged.remotes.delete(remote_name) + nil end - # Add a new remote to this repository. Returns a Rugged::Remote object + # Add a new remote to this repository. def remote_add(remote_name, url) rugged.remotes.create(remote_name, url) + nil end # Update the specified remote using the values in the +options+ hash @@ -610,6 +612,7 @@ module Gitlab def remote_update(remote_name, url:) # TODO: Implement other remote options rugged.remotes.set_url(remote_name, url) + nil end AUTOCRLF_VALUES = { -- cgit v1.2.1 From 539694a383bce9f02025c20aa777f283e106b158 Mon Sep 17 00:00:00 2001 From: haseeb Date: Fri, 18 Aug 2017 20:04:26 +0530 Subject: [skip ci] changelog and minor style changes added --- app/services/merge_requests/create_from_issue_service.rb | 2 +- changelogs/unreleased/35343-inherit-milestones-and-labels.yml | 5 +++++ spec/services/merge_requests/create_from_issue_service_spec.rb | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 changelogs/unreleased/35343-inherit-milestones-and-labels.yml diff --git a/app/services/merge_requests/create_from_issue_service.rb b/app/services/merge_requests/create_from_issue_service.rb index 6bc70d8175e..e9e4513e0d0 100644 --- a/app/services/merge_requests/create_from_issue_service.rb +++ b/app/services/merge_requests/create_from_issue_service.rb @@ -3,7 +3,7 @@ module MergeRequests def execute return error('Invalid issue iid') unless issue_iid.present? && issue.present? - params[:label_ids] = issue.label_ids if issue.label_ids.any? + params[:label_ids] = issue.label_ids if issue.label_ids.any? result = CreateBranchService.new(project, current_user).execute(branch_name, ref) return result if result[:status] == :error diff --git a/changelogs/unreleased/35343-inherit-milestones-and-labels.yml b/changelogs/unreleased/35343-inherit-milestones-and-labels.yml new file mode 100644 index 00000000000..ce737a67356 --- /dev/null +++ b/changelogs/unreleased/35343-inherit-milestones-and-labels.yml @@ -0,0 +1,5 @@ +--- +title: inherits milestone and labels when a merge request is created from issue +merge_request: 13461 +author: haseebeqx +type: added diff --git a/spec/services/merge_requests/create_from_issue_service_spec.rb b/spec/services/merge_requests/create_from_issue_service_spec.rb index b3c0e6518ef..313f87ae1f6 100644 --- a/spec/services/merge_requests/create_from_issue_service_spec.rb +++ b/spec/services/merge_requests/create_from_issue_service_spec.rb @@ -2,10 +2,10 @@ require 'spec_helper' describe MergeRequests::CreateFromIssueService do let(:project) { create(:project, :repository) } - let(:user) { create(:user) } + let(:user) { create(:user) } let(:label_ids) { create_pair(:label, project: project).map(&:id) } let(:milestone_id) { create(:milestone, project: project).id } - let(:issue) { create(:issue, project: project, milestone_id: milestone_id) } + let(:issue) { create(:issue, project: project, milestone_id: milestone_id) } subject(:service) { described_class.new(project, user, issue_iid: issue.iid) } -- cgit v1.2.1 From 998833dfdcf15a1e63f342efa6cba24bb370c610 Mon Sep 17 00:00:00 2001 From: Felipe Artur Date: Fri, 18 Aug 2017 13:02:05 -0300 Subject: Remove ignore columns --- app/models/issue.rb | 3 --- app/models/merge_request.rb | 1 - 2 files changed, 4 deletions(-) diff --git a/app/models/issue.rb b/app/models/issue.rb index 1c948c8957e..043da9967a1 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -9,11 +9,8 @@ class Issue < ActiveRecord::Base include Spammable include FasterCacheKeys include RelativePositioning - include IgnorableColumn include CreatedAtFilterable - ignore_column :position - DueDateStruct = Struct.new(:title, :name).freeze NoDueDate = DueDateStruct.new('No Due Date', '0').freeze AnyDueDate = DueDateStruct.new('Any Due Date', '').freeze diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index ac08dc0ee1f..f028d2395c1 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -7,7 +7,6 @@ class MergeRequest < ActiveRecord::Base include IgnorableColumn include CreatedAtFilterable - ignore_column :position ignore_column :locked_at belongs_to :target_project, class_name: "Project" -- cgit v1.2.1 From b789f3999630a0c3c9f6452ac27131b6c27da4f2 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 18 Aug 2017 18:16:01 +0200 Subject: Use Gitaly 0.33.0 --- GITALY_SERVER_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 9eb2aa3f109..be386c9ede3 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -0.32.0 +0.33.0 -- cgit v1.2.1 From 526a21698184706e3ed85e8a4c96213f101e2d5e Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 20:21:03 +0000 Subject: Delete issue_tracker.png --- doc/user/project/issues/img/issue_tracker.png | Bin 37037 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100755 doc/user/project/issues/img/issue_tracker.png diff --git a/doc/user/project/issues/img/issue_tracker.png b/doc/user/project/issues/img/issue_tracker.png deleted file mode 100755 index ab25cb64d13..00000000000 Binary files a/doc/user/project/issues/img/issue_tracker.png and /dev/null differ -- cgit v1.2.1 From d838f86ff84c321bed0d7d69ae3bc53b5c3c6be6 Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 20:23:12 +0000 Subject: group_issues_list_view.png --- doc/user/project/issues/img/group_issues_list_view.png | Bin 0 -> 545504 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 doc/user/project/issues/img/group_issues_list_view.png diff --git a/doc/user/project/issues/img/group_issues_list_view.png b/doc/user/project/issues/img/group_issues_list_view.png new file mode 100644 index 00000000000..c6a70e58a17 Binary files /dev/null and b/doc/user/project/issues/img/group_issues_list_view.png differ -- cgit v1.2.1 From 842b63481b2bc90cba1d2267a118dfb538af3a69 Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 20:23:20 +0000 Subject: Delete group_issues_list_view.png --- doc/user/project/issues/img/group_issues_list_view.png | Bin 545504 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 doc/user/project/issues/img/group_issues_list_view.png diff --git a/doc/user/project/issues/img/group_issues_list_view.png b/doc/user/project/issues/img/group_issues_list_view.png deleted file mode 100644 index c6a70e58a17..00000000000 Binary files a/doc/user/project/issues/img/group_issues_list_view.png and /dev/null differ -- cgit v1.2.1 From 8158f980aa7897aee0ca5202e942273541e29906 Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 20:24:47 +0000 Subject: group_issues_list_view.png --- doc/user/project/issues/img/group_issues_list_view.png | Bin 0 -> 265130 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 doc/user/project/issues/img/group_issues_list_view.png diff --git a/doc/user/project/issues/img/group_issues_list_view.png b/doc/user/project/issues/img/group_issues_list_view.png new file mode 100644 index 00000000000..5d20e8cbc89 Binary files /dev/null and b/doc/user/project/issues/img/group_issues_list_view.png differ -- cgit v1.2.1 From fe18ec58ec688f4e79eccdd946f98b7b659615ea Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 20:25:02 +0000 Subject: project_issues_list_view.png --- .../project/issues/img/project_issues_list_view.png | Bin 0 -> 309131 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 doc/user/project/issues/img/project_issues_list_view.png diff --git a/doc/user/project/issues/img/project_issues_list_view.png b/doc/user/project/issues/img/project_issues_list_view.png new file mode 100644 index 00000000000..2fcc9e8d9da Binary files /dev/null and b/doc/user/project/issues/img/project_issues_list_view.png differ -- cgit v1.2.1 From 8c650dc629ef4aac0180d5af53f63d948f08f4c0 Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 20:26:49 +0000 Subject: Update issue docs --- doc/user/project/issues/index.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index 1f78849a92c..dedd02396ab 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -1,13 +1,13 @@ # Issues -The GitLab Issue Tracker is an advanced and complete tool +GitLab issues is an advanced and complete tool for tracking the evolution of a new idea or the process of solving a problem. It allows you, your team, and your collaborators to share and discuss proposals before and while implementing them. -Issues and the GitLab Issue Tracker are available in all +GitLab issues is available in all [GitLab Products](https://about.gitlab.com/products/) as part of the [GitLab Workflow](https://about.gitlab.com/2016/10/25/gitlab-workflow-an-overview/). @@ -46,13 +46,20 @@ Create [issue templates](#issue-templates) to make collaboration consistent and containing all information you need. For example, you can create a template for feature proposals and another one for bug reports. -## Issue Tracker +## Issue list views, searching, and filtering -The issue tracker is the collection of opened and closed issues created in a project. +View all the issues within a project by navigating to **Project > Issues**. +There are tabs to quickly filter by open and closed issues. -![Issue tracker](img/issue_tracker.png) +![Project issues list view](img/project_issues_list_view.png) -Find the issue tracker by navigating to your **Project's Dashboard** > **Issues**. +View all the issues in a group (that is, all the issues across all projects in that +group) by navigating to **Group > Issues**. This view also has the open and closed +issue tabs. + +![Group Issues list view](img/group_issues_list_view.png) + +In both these list views, [search and filter the results](). ## GitLab Issues Functionalities -- cgit v1.2.1 From c32194dca0afed5c6e440ffe2e32d9bedc097ddb Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 20:32:20 +0000 Subject: group_merge_requests_list_view.png --- .../img/group_merge_requests_list_view.png | Bin 0 -> 283066 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 doc/user/project/merge_requests/img/group_merge_requests_list_view.png diff --git a/doc/user/project/merge_requests/img/group_merge_requests_list_view.png b/doc/user/project/merge_requests/img/group_merge_requests_list_view.png new file mode 100644 index 00000000000..02a88d0112f Binary files /dev/null and b/doc/user/project/merge_requests/img/group_merge_requests_list_view.png differ -- cgit v1.2.1 From ad99fe5df0f79eea421edbbea0887d81a5606f96 Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 20:32:37 +0000 Subject: project_merge_requests_list_view.png --- .../img/project_merge_requests_list_view.png | Bin 0 -> 325819 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 doc/user/project/merge_requests/img/project_merge_requests_list_view.png diff --git a/doc/user/project/merge_requests/img/project_merge_requests_list_view.png b/doc/user/project/merge_requests/img/project_merge_requests_list_view.png new file mode 100644 index 00000000000..702ec1a2949 Binary files /dev/null and b/doc/user/project/merge_requests/img/project_merge_requests_list_view.png differ -- cgit v1.2.1 From f77798ab0ddf83e468f1525f132f974500e4ccff Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 20:33:14 +0000 Subject: Merge request list views --- doc/user/project/merge_requests/index.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index 9bdf2a998d3..53f71e544ad 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -56,6 +56,22 @@ B. Consider you're a web developer writing a webpage for your company's: 1. Once approved, your merge request is [squashed and merged](https://docs.gitlab.com/ee/user/project/merge_requests/squash_and_merge.html), and [deployed to staging with GitLab Pages](https://about.gitlab.com/2016/08/26/ci-deployment-and-environments/) (Squash and Merge is available in GitLab Enterprise Edition Starter) 1. Your production team [cherry picks](#cherry-pick-changes) the merge commit into production + +## Merge request list views, searching, and filtering + +View all the merge requests within a project by navigating to **Project > Merge Requests**. +There are tabs to quickly filter by open, merged, and closed merge requests. + +![Project merge requests list view](img/project_merge_requests_list_view.png) + +View all the merge requests in a group (that is, all the merge requests across all projects in that +group) by navigating to **Group > Merge Requests**. This view also has the open, merged, and closed +merge request tabs. + +![Group Issues list view](img/group_merge_requests_list_view.png) + +In both these list views, [search and filter the results](). + ## Authorization for merge requests There are two main ways to have a merge request flow with GitLab: -- cgit v1.2.1 From 1939052afc8995123e67786ec92762e1a7d17fed Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 20:39:48 +0000 Subject: Update search docs --- doc/user/search/index.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/user/search/index.md b/doc/user/search/index.md index 79f34fd29ba..52af864d54e 100644 --- a/doc/user/search/index.md +++ b/doc/user/search/index.md @@ -48,6 +48,10 @@ the same way as you do for projects. ![filter issues in a group](img/group_issues_filter.png) +The same process is valid for merge requests. Navigate to your project's **Merge Requests** tab. +The search and filter UI currently uses dropdowns. In a future release, the same +dynamic UI as above will be carried over here. + ## Search history You can view recent searches by clicking on the little arrow-clock icon, which is to the left of the search input. Click the search entry to run that search again. This feature is available for issues and merge requests. Searches are stored locally in your browser. -- cgit v1.2.1 From 02c13cbb4d52cc83573519cf42ffa2e468016526 Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 20:48:01 +0000 Subject: Update search and filter links --- doc/use..r/project/merge_requests/index.md | 251 +++++++++++++++++++++++++++++ doc/user/project/merge_requests/index.md | 249 ---------------------------- 2 files changed, 251 insertions(+), 249 deletions(-) create mode 100644 doc/use..r/project/merge_requests/index.md delete mode 100644 doc/user/project/merge_requests/index.md diff --git a/doc/use..r/project/merge_requests/index.md b/doc/use..r/project/merge_requests/index.md new file mode 100644 index 00000000000..6c72f3e81b3 --- /dev/null +++ b/doc/use..r/project/merge_requests/index.md @@ -0,0 +1,251 @@ +# Merge requests + +Merge requests allow you to exchange changes you made to source code and +collaborate with other people on the same project. + +## Overview + +A Merge Request (**MR**) is the basis of GitLab as a code collaboration +and version control platform. +Is it simple as the name implies: a _request_ to _merge_ one branch into another. + +With GitLab merge requests, you can: + +- Compare the changes between two [branches](https://git-scm.com/book/en/v2/Git-Branching-Branches-in-a-Nutshell#_git_branching) +- [Review and discuss](../../discussions/index.md#discussions) the proposed modifications inline +- Live preview the changes when [Review Apps](../../../ci/review_apps/index.md) is configured for your project +- Build, test, and deploy your code in a per-branch basis with built-in [GitLab CI/CD](../../../ci/README.md) +- Prevent the merge request from being merged before it's ready with [WIP MRs](#work-in-progress-merge-requests) +- View the deployment process through [Pipeline Graphs](../../../ci/pipelines.md#pipeline-graphs) +- [Automatically close the issue(s)](../../project/issues/closing_issues.md#via-merge-request) that originated the implementation proposed in the merge request +- Assign it to any registered user, and change the assignee how many times you need +- Assign a [milestone](../../project/milestones/index.md) and track the development of a broader implementation +- Organize your issues and merge requests consistently throughout the project with [labels](../../project/labels.md) +- Add a time estimation and the time spent with that merge request with [Time Tracking](../../../workflow/time_tracking.html#time-tracking) +- [Resolve merge conflicts from the UI](#resolve-conflicts) + +With **[GitLab Enterprise Edition][ee]**, you can also: + +- View the deployment process across projects with [Multi-Project Pipeline Graphs](https://docs.gitlab.com/ee/ci/multi_project_pipeline_graphs.html#multi-project-pipeline-graphs) (available only in GitLab Enterprise Edition Premium) +- Request [approvals](https://docs.gitlab.com/ee/user/project/merge_requests/merge_request_approvals.html) from your managers (available in GitLab Enterprise Edition Starter) +- Enable [fast-forward merge requests](https://docs.gitlab.com/ee/user/project/merge_requests/fast_forward_merge.html) (available in GitLab Enterprise Edition Starter) +- [Squash and merge](https://docs.gitlab.com/ee/user/project/merge_requests/squash_and_merge.html) for a cleaner commit history (available in GitLab Enterprise Edition Starter) +- Enable [semi-linear history merge requests](https://docs.gitlab.com/ee/user/project/merge_requests/index.html#semi-linear-history-merge-requests) as another security layer to guarantee the pipeline is passing in the target branch (available in GitLab Enterprise Edition Starter) +- Analise the impact of your changes with [Code Quality reports](https://docs.gitlab.com/ee/user/project/merge_requests/code_quality_diff.html) (available in GitLab Enterprise Edition Starter) + +## Use cases + +A. Consider you are a software developer working in a team: + +1. You checkout a new branch, and submit your changes through a merge request +1. You gather feedback from your team +1. You work on the implementation optimizing code with [Code Quality reports](https://docs.gitlab.com/ee/user/project/merge_requests/code_quality_diff.html) (available in GitLab Enterprise Edition Starter) +1. You build and test your changes with GitLab CI/CD +1. You request the approval from your manager +1. Your manager pushes a commit with his final review, [approves the merge request](https://docs.gitlab.com/ee/user/project/merge_requests/merge_request_approvals.html), and set it to [merge when pipeline succeeds](#merge-when-pipeline-succeeds) (Merge Request Approvals are available in GitLab Enterprise Edition Starter) +1. Your changes get deployed to production with [manual actions](../../../ci/yaml/README.md#manual-actions) for GitLab CI/CD +1. Your implementations were successfully shipped to your customer + +B. Consider you're a web developer writing a webpage for your company's: + +1. You checkout a new branch, and submit a new page through a merge request +1. You gather feedback from your reviewers +1. Your changes are previewed with [Review Apps](../../../ci/review_apps/index.md) +1. You request your web designers for their implementation +1. You request the [approval](https://docs.gitlab.com/ee/user/project/merge_requests/merge_request_approvals.html) from your manager (available in GitLab Enterprise Edition Starter) +1. Once approved, your merge request is [squashed and merged](https://docs.gitlab.com/ee/user/project/merge_requests/squash_and_merge.html), and [deployed to staging with GitLab Pages](https://about.gitlab.com/2016/08/26/ci-deployment-and-environments/) (Squash and Merge is available in GitLab Enterprise Edition Starter) +1. Your production team [cherry picks](#cherry-pick-changes) the merge commit into production + + +## Merge request list views, searching, and filtering + +View all the merge requests within a project by navigating to **Project > Merge Requests**. +There are tabs to quickly filter by open, merged, and closed merge requests. + +![Project merge requests list view](img/project_merge_requests_list_view.png) + +[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-project) + +View all the merge requests in a group (that is, all the merge requests across all projects in that +group) by navigating to **Group > Merge Requests**. This view also has the open, merged, and closed +merge request tabs. + +![Group Issues list view](img/group_merge_requests_list_view.png) + +[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-group) + +## Authorization for merge requests + +There are two main ways to have a merge request flow with GitLab: + +1. Working with [protected branches][] in a single repository +1. Working with forks of an authoritative project + +[Learn more about the authorization for merge requests.](authorization_for_merge_requests.md) + +## Cherry-pick changes + +Cherry-pick any commit in the UI by simply clicking the **Cherry-pick** button +in a merged merge requests or a commit. + +[Learn more about cherry-picking changes.](cherry_pick_changes.md) + +## Merge when pipeline succeeds + +When reviewing a merge request that looks ready to merge but still has one or +more CI jobs running, you can set it to be merged automatically when CI +pipeline succeeds. This way, you don't have to wait for the pipeline to finish +and remember to merge the request manually. + +[Learn more about merging when pipeline succeeds.](merge_when_pipeline_succeeds.md) + +## Resolve discussion comments in merge requests reviews + +Keep track of the progress during a code review with resolving comments. +Resolving comments prevents you from forgetting to address feedback and lets +you hide discussions that are no longer relevant. + +[Read more about resolving discussion comments in merge requests reviews.](../../discussions/index.md) + +## Resolve conflicts + +When a merge request has conflicts, GitLab may provide the option to resolve +those conflicts in the GitLab UI. + +[Learn more about resolving merge conflicts in the UI.](resolve_conflicts.md) + +## Revert changes + +GitLab implements Git's powerful feature to revert any commit with introducing +a **Revert** button in merge requests and commit details. + +[Learn more about reverting changes in the UI](revert_changes.md) + +## Merge requests versions + +Every time you push to a branch that is tied to a merge request, a new version +of merge request diff is created. When you visit a merge request that contains +more than one pushes, you can select and compare the versions of those merge +request diffs. + +[Read more about the merge requests versions.](versions.md) + +## Work In Progress merge requests + +To prevent merge requests from accidentally being accepted before they're +completely ready, GitLab blocks the "Accept" button for merge requests that +have been marked as a **Work In Progress**. + +[Learn more about settings a merge request as "Work In Progress".](work_in_progress_merge_requests.md) + +## Ignore whitespace changes in Merge Request diff view + +If you click the **Hide whitespace changes** button, you can see the diff +without whitespace changes (if there are any). This is also working when on a +specific commit page. + +![MR diff](img/merge_request_diff.png) + +>**Tip:** +You can append `?w=1` while on the diffs page of a merge request to ignore any +whitespace changes. + +## Live preview with Review Apps + +If you configured [Review Apps](https://about.gitlab.com/features/review-apps/) for your project, +you can preview the changes submitted to a feature-branch through a merge request +in a per-branch basis. No need to checkout the branch, install and preview locally; +all your changes will be available to preview by anyone with the Review Apps link. + +[Read more about Review Apps.](../../../ci/review_apps/index.md) + + +## Tips + +Here are some tips that will help you be more efficient with merge requests in +the command line. + +> **Note:** +This section might move in its own document in the future. + +### Checkout merge requests locally + +A merge request contains all the history from a repository, plus the additional +commits added to the branch associated with the merge request. Here's a few +tricks to checkout a merge request locally. + +Please note that you can checkout a merge request locally even if the source +project is a fork (even a private fork) of the target project. + +#### Checkout locally by adding a git alias + +Add the following alias to your `~/.gitconfig`: + +``` +[alias] + mr = !sh -c 'git fetch $1 merge-requests/$2/head:mr-$1-$2 && git checkout mr-$1-$2' - +``` + +Now you can check out a particular merge request from any repository and any +remote. For example, to check out the merge request with ID 5 as shown in GitLab +from the `upstream` remote, do: + +``` +git mr upstream 5 +``` + +This will fetch the merge request into a local `mr-upstream-5` branch and check +it out. + +#### Checkout locally by modifying `.git/config` for a given repository + +Locate the section for your GitLab remote in the `.git/config` file. It looks +like this: + +``` +[remote "origin"] + url = https://gitlab.com/gitlab-org/gitlab-ce.git + fetch = +refs/heads/*:refs/remotes/origin/* +``` + +You can open the file with: + +``` +git config -e +``` + +Now add the following line to the above section: + +``` +fetch = +refs/merge-requests/*/head:refs/remotes/origin/merge-requests/* +``` + +In the end, it should look like this: + +``` +[remote "origin"] + url = https://gitlab.com/gitlab-org/gitlab-ce.git + fetch = +refs/heads/*:refs/remotes/origin/* + fetch = +refs/merge-requests/*/head:refs/remotes/origin/merge-requests/* +``` + +Now you can fetch all the merge requests: + +``` +git fetch origin + +... +From https://gitlab.com/gitlab-org/gitlab-ce.git + * [new ref] refs/merge-requests/1/head -> origin/merge-requests/1 + * [new ref] refs/merge-requests/2/head -> origin/merge-requests/2 +... +``` + +And to check out a particular merge request: + +``` +git checkout origin/merge-requests/1 +``` + +[protected branches]: ../protected_branches.md +[ee]: https://about.gitlab.com/gitlab-ee/ "GitLab Enterprise Edition" diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md deleted file mode 100644 index 53f71e544ad..00000000000 --- a/doc/user/project/merge_requests/index.md +++ /dev/null @@ -1,249 +0,0 @@ -# Merge requests - -Merge requests allow you to exchange changes you made to source code and -collaborate with other people on the same project. - -## Overview - -A Merge Request (**MR**) is the basis of GitLab as a code collaboration -and version control platform. -Is it simple as the name implies: a _request_ to _merge_ one branch into another. - -With GitLab merge requests, you can: - -- Compare the changes between two [branches](https://git-scm.com/book/en/v2/Git-Branching-Branches-in-a-Nutshell#_git_branching) -- [Review and discuss](../../discussions/index.md#discussions) the proposed modifications inline -- Live preview the changes when [Review Apps](../../../ci/review_apps/index.md) is configured for your project -- Build, test, and deploy your code in a per-branch basis with built-in [GitLab CI/CD](../../../ci/README.md) -- Prevent the merge request from being merged before it's ready with [WIP MRs](#work-in-progress-merge-requests) -- View the deployment process through [Pipeline Graphs](../../../ci/pipelines.md#pipeline-graphs) -- [Automatically close the issue(s)](../../project/issues/closing_issues.md#via-merge-request) that originated the implementation proposed in the merge request -- Assign it to any registered user, and change the assignee how many times you need -- Assign a [milestone](../../project/milestones/index.md) and track the development of a broader implementation -- Organize your issues and merge requests consistently throughout the project with [labels](../../project/labels.md) -- Add a time estimation and the time spent with that merge request with [Time Tracking](../../../workflow/time_tracking.html#time-tracking) -- [Resolve merge conflicts from the UI](#resolve-conflicts) - -With **[GitLab Enterprise Edition][ee]**, you can also: - -- View the deployment process across projects with [Multi-Project Pipeline Graphs](https://docs.gitlab.com/ee/ci/multi_project_pipeline_graphs.html#multi-project-pipeline-graphs) (available only in GitLab Enterprise Edition Premium) -- Request [approvals](https://docs.gitlab.com/ee/user/project/merge_requests/merge_request_approvals.html) from your managers (available in GitLab Enterprise Edition Starter) -- Enable [fast-forward merge requests](https://docs.gitlab.com/ee/user/project/merge_requests/fast_forward_merge.html) (available in GitLab Enterprise Edition Starter) -- [Squash and merge](https://docs.gitlab.com/ee/user/project/merge_requests/squash_and_merge.html) for a cleaner commit history (available in GitLab Enterprise Edition Starter) -- Enable [semi-linear history merge requests](https://docs.gitlab.com/ee/user/project/merge_requests/index.html#semi-linear-history-merge-requests) as another security layer to guarantee the pipeline is passing in the target branch (available in GitLab Enterprise Edition Starter) -- Analise the impact of your changes with [Code Quality reports](https://docs.gitlab.com/ee/user/project/merge_requests/code_quality_diff.html) (available in GitLab Enterprise Edition Starter) - -## Use cases - -A. Consider you are a software developer working in a team: - -1. You checkout a new branch, and submit your changes through a merge request -1. You gather feedback from your team -1. You work on the implementation optimizing code with [Code Quality reports](https://docs.gitlab.com/ee/user/project/merge_requests/code_quality_diff.html) (available in GitLab Enterprise Edition Starter) -1. You build and test your changes with GitLab CI/CD -1. You request the approval from your manager -1. Your manager pushes a commit with his final review, [approves the merge request](https://docs.gitlab.com/ee/user/project/merge_requests/merge_request_approvals.html), and set it to [merge when pipeline succeeds](#merge-when-pipeline-succeeds) (Merge Request Approvals are available in GitLab Enterprise Edition Starter) -1. Your changes get deployed to production with [manual actions](../../../ci/yaml/README.md#manual-actions) for GitLab CI/CD -1. Your implementations were successfully shipped to your customer - -B. Consider you're a web developer writing a webpage for your company's: - -1. You checkout a new branch, and submit a new page through a merge request -1. You gather feedback from your reviewers -1. Your changes are previewed with [Review Apps](../../../ci/review_apps/index.md) -1. You request your web designers for their implementation -1. You request the [approval](https://docs.gitlab.com/ee/user/project/merge_requests/merge_request_approvals.html) from your manager (available in GitLab Enterprise Edition Starter) -1. Once approved, your merge request is [squashed and merged](https://docs.gitlab.com/ee/user/project/merge_requests/squash_and_merge.html), and [deployed to staging with GitLab Pages](https://about.gitlab.com/2016/08/26/ci-deployment-and-environments/) (Squash and Merge is available in GitLab Enterprise Edition Starter) -1. Your production team [cherry picks](#cherry-pick-changes) the merge commit into production - - -## Merge request list views, searching, and filtering - -View all the merge requests within a project by navigating to **Project > Merge Requests**. -There are tabs to quickly filter by open, merged, and closed merge requests. - -![Project merge requests list view](img/project_merge_requests_list_view.png) - -View all the merge requests in a group (that is, all the merge requests across all projects in that -group) by navigating to **Group > Merge Requests**. This view also has the open, merged, and closed -merge request tabs. - -![Group Issues list view](img/group_merge_requests_list_view.png) - -In both these list views, [search and filter the results](). - -## Authorization for merge requests - -There are two main ways to have a merge request flow with GitLab: - -1. Working with [protected branches][] in a single repository -1. Working with forks of an authoritative project - -[Learn more about the authorization for merge requests.](authorization_for_merge_requests.md) - -## Cherry-pick changes - -Cherry-pick any commit in the UI by simply clicking the **Cherry-pick** button -in a merged merge requests or a commit. - -[Learn more about cherry-picking changes.](cherry_pick_changes.md) - -## Merge when pipeline succeeds - -When reviewing a merge request that looks ready to merge but still has one or -more CI jobs running, you can set it to be merged automatically when CI -pipeline succeeds. This way, you don't have to wait for the pipeline to finish -and remember to merge the request manually. - -[Learn more about merging when pipeline succeeds.](merge_when_pipeline_succeeds.md) - -## Resolve discussion comments in merge requests reviews - -Keep track of the progress during a code review with resolving comments. -Resolving comments prevents you from forgetting to address feedback and lets -you hide discussions that are no longer relevant. - -[Read more about resolving discussion comments in merge requests reviews.](../../discussions/index.md) - -## Resolve conflicts - -When a merge request has conflicts, GitLab may provide the option to resolve -those conflicts in the GitLab UI. - -[Learn more about resolving merge conflicts in the UI.](resolve_conflicts.md) - -## Revert changes - -GitLab implements Git's powerful feature to revert any commit with introducing -a **Revert** button in merge requests and commit details. - -[Learn more about reverting changes in the UI](revert_changes.md) - -## Merge requests versions - -Every time you push to a branch that is tied to a merge request, a new version -of merge request diff is created. When you visit a merge request that contains -more than one pushes, you can select and compare the versions of those merge -request diffs. - -[Read more about the merge requests versions.](versions.md) - -## Work In Progress merge requests - -To prevent merge requests from accidentally being accepted before they're -completely ready, GitLab blocks the "Accept" button for merge requests that -have been marked as a **Work In Progress**. - -[Learn more about settings a merge request as "Work In Progress".](work_in_progress_merge_requests.md) - -## Ignore whitespace changes in Merge Request diff view - -If you click the **Hide whitespace changes** button, you can see the diff -without whitespace changes (if there are any). This is also working when on a -specific commit page. - -![MR diff](img/merge_request_diff.png) - ->**Tip:** -You can append `?w=1` while on the diffs page of a merge request to ignore any -whitespace changes. - -## Live preview with Review Apps - -If you configured [Review Apps](https://about.gitlab.com/features/review-apps/) for your project, -you can preview the changes submitted to a feature-branch through a merge request -in a per-branch basis. No need to checkout the branch, install and preview locally; -all your changes will be available to preview by anyone with the Review Apps link. - -[Read more about Review Apps.](../../../ci/review_apps/index.md) - - -## Tips - -Here are some tips that will help you be more efficient with merge requests in -the command line. - -> **Note:** -This section might move in its own document in the future. - -### Checkout merge requests locally - -A merge request contains all the history from a repository, plus the additional -commits added to the branch associated with the merge request. Here's a few -tricks to checkout a merge request locally. - -Please note that you can checkout a merge request locally even if the source -project is a fork (even a private fork) of the target project. - -#### Checkout locally by adding a git alias - -Add the following alias to your `~/.gitconfig`: - -``` -[alias] - mr = !sh -c 'git fetch $1 merge-requests/$2/head:mr-$1-$2 && git checkout mr-$1-$2' - -``` - -Now you can check out a particular merge request from any repository and any -remote. For example, to check out the merge request with ID 5 as shown in GitLab -from the `upstream` remote, do: - -``` -git mr upstream 5 -``` - -This will fetch the merge request into a local `mr-upstream-5` branch and check -it out. - -#### Checkout locally by modifying `.git/config` for a given repository - -Locate the section for your GitLab remote in the `.git/config` file. It looks -like this: - -``` -[remote "origin"] - url = https://gitlab.com/gitlab-org/gitlab-ce.git - fetch = +refs/heads/*:refs/remotes/origin/* -``` - -You can open the file with: - -``` -git config -e -``` - -Now add the following line to the above section: - -``` -fetch = +refs/merge-requests/*/head:refs/remotes/origin/merge-requests/* -``` - -In the end, it should look like this: - -``` -[remote "origin"] - url = https://gitlab.com/gitlab-org/gitlab-ce.git - fetch = +refs/heads/*:refs/remotes/origin/* - fetch = +refs/merge-requests/*/head:refs/remotes/origin/merge-requests/* -``` - -Now you can fetch all the merge requests: - -``` -git fetch origin - -... -From https://gitlab.com/gitlab-org/gitlab-ce.git - * [new ref] refs/merge-requests/1/head -> origin/merge-requests/1 - * [new ref] refs/merge-requests/2/head -> origin/merge-requests/2 -... -``` - -And to check out a particular merge request: - -``` -git checkout origin/merge-requests/1 -``` - -[protected branches]: ../protected_branches.md -[ee]: https://about.gitlab.com/gitlab-ee/ "GitLab Enterprise Edition" -- cgit v1.2.1 From fb8dd2f418a90fea41182497b34e7c17aa0955b0 Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 20:51:09 +0000 Subject: Search and filter results. --- doc/user/project/issues/index.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index dedd02396ab..759744881f1 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -53,13 +53,15 @@ There are tabs to quickly filter by open and closed issues. ![Project issues list view](img/project_issues_list_view.png) +[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-project). + View all the issues in a group (that is, all the issues across all projects in that group) by navigating to **Group > Issues**. This view also has the open and closed issue tabs. ![Group Issues list view](img/group_issues_list_view.png) -In both these list views, [search and filter the results](). +[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-group). ## GitLab Issues Functionalities -- cgit v1.2.1 From f281b7d7ac1bad85f0eb479a836320abcd598e5b Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 20:54:26 +0000 Subject: Punctuation. --- doc/use..r/project/merge_requests/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/use..r/project/merge_requests/index.md b/doc/use..r/project/merge_requests/index.md index 6c72f3e81b3..b2fe8f21f8b 100644 --- a/doc/use..r/project/merge_requests/index.md +++ b/doc/use..r/project/merge_requests/index.md @@ -64,7 +64,7 @@ There are tabs to quickly filter by open, merged, and closed merge requests. ![Project merge requests list view](img/project_merge_requests_list_view.png) -[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-project) +[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-project). View all the merge requests in a group (that is, all the merge requests across all projects in that group) by navigating to **Group > Merge Requests**. This view also has the open, merged, and closed @@ -72,7 +72,7 @@ merge request tabs. ![Group Issues list view](img/group_merge_requests_list_view.png) -[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-group) +[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-group). ## Authorization for merge requests -- cgit v1.2.1 From ed32b920531c5ca33fadb9bcfa8cd68df88baeaf Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 21:31:58 +0000 Subject: Delete index.md --- doc/use..r/project/merge_requests/index.md | 251 ----------------------------- 1 file changed, 251 deletions(-) delete mode 100644 doc/use..r/project/merge_requests/index.md diff --git a/doc/use..r/project/merge_requests/index.md b/doc/use..r/project/merge_requests/index.md deleted file mode 100644 index b2fe8f21f8b..00000000000 --- a/doc/use..r/project/merge_requests/index.md +++ /dev/null @@ -1,251 +0,0 @@ -# Merge requests - -Merge requests allow you to exchange changes you made to source code and -collaborate with other people on the same project. - -## Overview - -A Merge Request (**MR**) is the basis of GitLab as a code collaboration -and version control platform. -Is it simple as the name implies: a _request_ to _merge_ one branch into another. - -With GitLab merge requests, you can: - -- Compare the changes between two [branches](https://git-scm.com/book/en/v2/Git-Branching-Branches-in-a-Nutshell#_git_branching) -- [Review and discuss](../../discussions/index.md#discussions) the proposed modifications inline -- Live preview the changes when [Review Apps](../../../ci/review_apps/index.md) is configured for your project -- Build, test, and deploy your code in a per-branch basis with built-in [GitLab CI/CD](../../../ci/README.md) -- Prevent the merge request from being merged before it's ready with [WIP MRs](#work-in-progress-merge-requests) -- View the deployment process through [Pipeline Graphs](../../../ci/pipelines.md#pipeline-graphs) -- [Automatically close the issue(s)](../../project/issues/closing_issues.md#via-merge-request) that originated the implementation proposed in the merge request -- Assign it to any registered user, and change the assignee how many times you need -- Assign a [milestone](../../project/milestones/index.md) and track the development of a broader implementation -- Organize your issues and merge requests consistently throughout the project with [labels](../../project/labels.md) -- Add a time estimation and the time spent with that merge request with [Time Tracking](../../../workflow/time_tracking.html#time-tracking) -- [Resolve merge conflicts from the UI](#resolve-conflicts) - -With **[GitLab Enterprise Edition][ee]**, you can also: - -- View the deployment process across projects with [Multi-Project Pipeline Graphs](https://docs.gitlab.com/ee/ci/multi_project_pipeline_graphs.html#multi-project-pipeline-graphs) (available only in GitLab Enterprise Edition Premium) -- Request [approvals](https://docs.gitlab.com/ee/user/project/merge_requests/merge_request_approvals.html) from your managers (available in GitLab Enterprise Edition Starter) -- Enable [fast-forward merge requests](https://docs.gitlab.com/ee/user/project/merge_requests/fast_forward_merge.html) (available in GitLab Enterprise Edition Starter) -- [Squash and merge](https://docs.gitlab.com/ee/user/project/merge_requests/squash_and_merge.html) for a cleaner commit history (available in GitLab Enterprise Edition Starter) -- Enable [semi-linear history merge requests](https://docs.gitlab.com/ee/user/project/merge_requests/index.html#semi-linear-history-merge-requests) as another security layer to guarantee the pipeline is passing in the target branch (available in GitLab Enterprise Edition Starter) -- Analise the impact of your changes with [Code Quality reports](https://docs.gitlab.com/ee/user/project/merge_requests/code_quality_diff.html) (available in GitLab Enterprise Edition Starter) - -## Use cases - -A. Consider you are a software developer working in a team: - -1. You checkout a new branch, and submit your changes through a merge request -1. You gather feedback from your team -1. You work on the implementation optimizing code with [Code Quality reports](https://docs.gitlab.com/ee/user/project/merge_requests/code_quality_diff.html) (available in GitLab Enterprise Edition Starter) -1. You build and test your changes with GitLab CI/CD -1. You request the approval from your manager -1. Your manager pushes a commit with his final review, [approves the merge request](https://docs.gitlab.com/ee/user/project/merge_requests/merge_request_approvals.html), and set it to [merge when pipeline succeeds](#merge-when-pipeline-succeeds) (Merge Request Approvals are available in GitLab Enterprise Edition Starter) -1. Your changes get deployed to production with [manual actions](../../../ci/yaml/README.md#manual-actions) for GitLab CI/CD -1. Your implementations were successfully shipped to your customer - -B. Consider you're a web developer writing a webpage for your company's: - -1. You checkout a new branch, and submit a new page through a merge request -1. You gather feedback from your reviewers -1. Your changes are previewed with [Review Apps](../../../ci/review_apps/index.md) -1. You request your web designers for their implementation -1. You request the [approval](https://docs.gitlab.com/ee/user/project/merge_requests/merge_request_approvals.html) from your manager (available in GitLab Enterprise Edition Starter) -1. Once approved, your merge request is [squashed and merged](https://docs.gitlab.com/ee/user/project/merge_requests/squash_and_merge.html), and [deployed to staging with GitLab Pages](https://about.gitlab.com/2016/08/26/ci-deployment-and-environments/) (Squash and Merge is available in GitLab Enterprise Edition Starter) -1. Your production team [cherry picks](#cherry-pick-changes) the merge commit into production - - -## Merge request list views, searching, and filtering - -View all the merge requests within a project by navigating to **Project > Merge Requests**. -There are tabs to quickly filter by open, merged, and closed merge requests. - -![Project merge requests list view](img/project_merge_requests_list_view.png) - -[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-project). - -View all the merge requests in a group (that is, all the merge requests across all projects in that -group) by navigating to **Group > Merge Requests**. This view also has the open, merged, and closed -merge request tabs. - -![Group Issues list view](img/group_merge_requests_list_view.png) - -[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-group). - -## Authorization for merge requests - -There are two main ways to have a merge request flow with GitLab: - -1. Working with [protected branches][] in a single repository -1. Working with forks of an authoritative project - -[Learn more about the authorization for merge requests.](authorization_for_merge_requests.md) - -## Cherry-pick changes - -Cherry-pick any commit in the UI by simply clicking the **Cherry-pick** button -in a merged merge requests or a commit. - -[Learn more about cherry-picking changes.](cherry_pick_changes.md) - -## Merge when pipeline succeeds - -When reviewing a merge request that looks ready to merge but still has one or -more CI jobs running, you can set it to be merged automatically when CI -pipeline succeeds. This way, you don't have to wait for the pipeline to finish -and remember to merge the request manually. - -[Learn more about merging when pipeline succeeds.](merge_when_pipeline_succeeds.md) - -## Resolve discussion comments in merge requests reviews - -Keep track of the progress during a code review with resolving comments. -Resolving comments prevents you from forgetting to address feedback and lets -you hide discussions that are no longer relevant. - -[Read more about resolving discussion comments in merge requests reviews.](../../discussions/index.md) - -## Resolve conflicts - -When a merge request has conflicts, GitLab may provide the option to resolve -those conflicts in the GitLab UI. - -[Learn more about resolving merge conflicts in the UI.](resolve_conflicts.md) - -## Revert changes - -GitLab implements Git's powerful feature to revert any commit with introducing -a **Revert** button in merge requests and commit details. - -[Learn more about reverting changes in the UI](revert_changes.md) - -## Merge requests versions - -Every time you push to a branch that is tied to a merge request, a new version -of merge request diff is created. When you visit a merge request that contains -more than one pushes, you can select and compare the versions of those merge -request diffs. - -[Read more about the merge requests versions.](versions.md) - -## Work In Progress merge requests - -To prevent merge requests from accidentally being accepted before they're -completely ready, GitLab blocks the "Accept" button for merge requests that -have been marked as a **Work In Progress**. - -[Learn more about settings a merge request as "Work In Progress".](work_in_progress_merge_requests.md) - -## Ignore whitespace changes in Merge Request diff view - -If you click the **Hide whitespace changes** button, you can see the diff -without whitespace changes (if there are any). This is also working when on a -specific commit page. - -![MR diff](img/merge_request_diff.png) - ->**Tip:** -You can append `?w=1` while on the diffs page of a merge request to ignore any -whitespace changes. - -## Live preview with Review Apps - -If you configured [Review Apps](https://about.gitlab.com/features/review-apps/) for your project, -you can preview the changes submitted to a feature-branch through a merge request -in a per-branch basis. No need to checkout the branch, install and preview locally; -all your changes will be available to preview by anyone with the Review Apps link. - -[Read more about Review Apps.](../../../ci/review_apps/index.md) - - -## Tips - -Here are some tips that will help you be more efficient with merge requests in -the command line. - -> **Note:** -This section might move in its own document in the future. - -### Checkout merge requests locally - -A merge request contains all the history from a repository, plus the additional -commits added to the branch associated with the merge request. Here's a few -tricks to checkout a merge request locally. - -Please note that you can checkout a merge request locally even if the source -project is a fork (even a private fork) of the target project. - -#### Checkout locally by adding a git alias - -Add the following alias to your `~/.gitconfig`: - -``` -[alias] - mr = !sh -c 'git fetch $1 merge-requests/$2/head:mr-$1-$2 && git checkout mr-$1-$2' - -``` - -Now you can check out a particular merge request from any repository and any -remote. For example, to check out the merge request with ID 5 as shown in GitLab -from the `upstream` remote, do: - -``` -git mr upstream 5 -``` - -This will fetch the merge request into a local `mr-upstream-5` branch and check -it out. - -#### Checkout locally by modifying `.git/config` for a given repository - -Locate the section for your GitLab remote in the `.git/config` file. It looks -like this: - -``` -[remote "origin"] - url = https://gitlab.com/gitlab-org/gitlab-ce.git - fetch = +refs/heads/*:refs/remotes/origin/* -``` - -You can open the file with: - -``` -git config -e -``` - -Now add the following line to the above section: - -``` -fetch = +refs/merge-requests/*/head:refs/remotes/origin/merge-requests/* -``` - -In the end, it should look like this: - -``` -[remote "origin"] - url = https://gitlab.com/gitlab-org/gitlab-ce.git - fetch = +refs/heads/*:refs/remotes/origin/* - fetch = +refs/merge-requests/*/head:refs/remotes/origin/merge-requests/* -``` - -Now you can fetch all the merge requests: - -``` -git fetch origin - -... -From https://gitlab.com/gitlab-org/gitlab-ce.git - * [new ref] refs/merge-requests/1/head -> origin/merge-requests/1 - * [new ref] refs/merge-requests/2/head -> origin/merge-requests/2 -... -``` - -And to check out a particular merge request: - -``` -git checkout origin/merge-requests/1 -``` - -[protected branches]: ../protected_branches.md -[ee]: https://about.gitlab.com/gitlab-ee/ "GitLab Enterprise Edition" -- cgit v1.2.1 From 411f341c4a25c1dd1defd73d562124ef1e8753b9 Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Fri, 18 Aug 2017 21:34:02 +0000 Subject: Merge requests search and filtering --- doc/user/project/merge_requests/index.md | 251 +++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 doc/user/project/merge_requests/index.md diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md new file mode 100644 index 00000000000..cd52471922f --- /dev/null +++ b/doc/user/project/merge_requests/index.md @@ -0,0 +1,251 @@ +# Merge requests + +Merge requests allow you to exchange changes you made to source code and +collaborate with other people on the same project. + +## Overview + +A Merge Request (**MR**) is the basis of GitLab as a code collaboration +and version control platform. +Is it simple as the name implies: a _request_ to _merge_ one branch into another. + +With GitLab merge requests, you can: + +- Compare the changes between two [branches](https://git-scm.com/book/en/v2/Git-Branching-Branches-in-a-Nutshell#_git_branching) +- [Review and discuss](../../discussions/index.md#discussions) the proposed modifications inline +- Live preview the changes when [Review Apps](../../../ci/review_apps/index.md) is configured for your project +- Build, test, and deploy your code in a per-branch basis with built-in [GitLab CI/CD](../../../ci/README.md) +- Prevent the merge request from being merged before it's ready with [WIP MRs](#work-in-progress-merge-requests) +- View the deployment process through [Pipeline Graphs](../../../ci/pipelines.md#pipeline-graphs) +- [Automatically close the issue(s)](../../project/issues/closing_issues.md#via-merge-request) that originated the implementation proposed in the merge request +- Assign it to any registered user, and change the assignee how many times you need +- Assign a [milestone](../../project/milestones/index.md) and track the development of a broader implementation +- Organize your issues and merge requests consistently throughout the project with [labels](../../project/labels.md) +- Add a time estimation and the time spent with that merge request with [Time Tracking](../../../workflow/time_tracking.html#time-tracking) +- [Resolve merge conflicts from the UI](#resolve-conflicts) + +With **[GitLab Enterprise Edition][ee]**, you can also: + +- View the deployment process across projects with [Multi-Project Pipeline Graphs](https://docs.gitlab.com/ee/ci/multi_project_pipeline_graphs.html#multi-project-pipeline-graphs) (available only in GitLab Enterprise Edition Premium) +- Request [approvals](https://docs.gitlab.com/ee/user/project/merge_requests/merge_request_approvals.html) from your managers (available in GitLab Enterprise Edition Starter) +- Enable [fast-forward merge requests](https://docs.gitlab.com/ee/user/project/merge_requests/fast_forward_merge.html) (available in GitLab Enterprise Edition Starter) +- [Squash and merge](https://docs.gitlab.com/ee/user/project/merge_requests/squash_and_merge.html) for a cleaner commit history (available in GitLab Enterprise Edition Starter) +- Enable [semi-linear history merge requests](https://docs.gitlab.com/ee/user/project/merge_requests/index.html#semi-linear-history-merge-requests) as another security layer to guarantee the pipeline is passing in the target branch (available in GitLab Enterprise Edition Starter) +- Analise the impact of your changes with [Code Quality reports](https://docs.gitlab.com/ee/user/project/merge_requests/code_quality_diff.html) (available in GitLab Enterprise Edition Starter) + +## Use cases + +A. Consider you are a software developer working in a team: + +1. You checkout a new branch, and submit your changes through a merge request +1. You gather feedback from your team +1. You work on the implementation optimizing code with [Code Quality reports](https://docs.gitlab.com/ee/user/project/merge_requests/code_quality_diff.html) (available in GitLab Enterprise Edition Starter) +1. You build and test your changes with GitLab CI/CD +1. You request the approval from your manager +1. Your manager pushes a commit with his final review, [approves the merge request](https://docs.gitlab.com/ee/user/project/merge_requests/merge_request_approvals.html), and set it to [merge when pipeline succeeds](#merge-when-pipeline-succeeds) (Merge Request Approvals are available in GitLab Enterprise Edition Starter) +1. Your changes get deployed to production with [manual actions](../../../ci/yaml/README.md#manual-actions) for GitLab CI/CD +1. Your implementations were successfully shipped to your customer + +B. Consider you're a web developer writing a webpage for your company's: + +1. You checkout a new branch, and submit a new page through a merge request +1. You gather feedback from your reviewers +1. Your changes are previewed with [Review Apps](../../../ci/review_apps/index.md) +1. You request your web designers for their implementation +1. You request the [approval](https://docs.gitlab.com/ee/user/project/merge_requests/merge_request_approvals.html) from your manager (available in GitLab Enterprise Edition Starter) +1. Once approved, your merge request is [squashed and merged](https://docs.gitlab.com/ee/user/project/merge_requests/squash_and_merge.html), and [deployed to staging with GitLab Pages](https://about.gitlab.com/2016/08/26/ci-deployment-and-environments/) (Squash and Merge is available in GitLab Enterprise Edition Starter) +1. Your production team [cherry picks](#cherry-pick-changes) the merge commit into production + + +## Merge request list views, searching, and filtering + +View all the merge requests within a project by navigating to **Project > Merge Requests**. +There are tabs to quickly filter by open, merged, and closed merge requests. + +![Project merge requests list view](img/project_merge_requests_list_view.png) + +[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-project). + +View all the merge requests in a group (that is, all the merge requests across all projects in that +group) by navigating to **Group > Merge Requests**. This view also has the open, merged, and closed +merge request tabs. + +![Group Issues list view](img/group_merge_requests_list_view.png) + +[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-group). + +## Authorization for merge requests + +There are two main ways to have a merge request flow with GitLab: + +1. Working with [protected branches][] in a single repository +1. Working with forks of an authoritative project + +[Learn more about the authorization for merge requests.](authorization_for_merge_requests.md) + +## Cherry-pick changes + +Cherry-pick any commit in the UI by simply clicking the **Cherry-pick** button +in a merged merge requests or a commit. + +[Learn more about cherry-picking changes.](cherry_pick_changes.md) + +## Merge when pipeline succeeds + +When reviewing a merge request that looks ready to merge but still has one or +more CI jobs running, you can set it to be merged automatically when CI +pipeline succeeds. This way, you don't have to wait for the pipeline to finish +and remember to merge the request manually. + +[Learn more about merging when pipeline succeeds.](merge_when_pipeline_succeeds.md) + +## Resolve discussion comments in merge requests reviews + +Keep track of the progress during a code review with resolving comments. +Resolving comments prevents you from forgetting to address feedback and lets +you hide discussions that are no longer relevant. + +[Read more about resolving discussion comments in merge requests reviews.](../../discussions/index.md) + +## Resolve conflicts + +When a merge request has conflicts, GitLab may provide the option to resolve +those conflicts in the GitLab UI. + +[Learn more about resolving merge conflicts in the UI.](resolve_conflicts.md) + +## Revert changes + +GitLab implements Git's powerful feature to revert any commit with introducing +a **Revert** button in merge requests and commit details. + +[Learn more about reverting changes in the UI](revert_changes.md) + +## Merge requests versions + +Every time you push to a branch that is tied to a merge request, a new version +of merge request diff is created. When you visit a merge request that contains +more than one pushes, you can select and compare the versions of those merge +request diffs. + +[Read more about the merge requests versions.](versions.md) + +## Work In Progress merge requests + +To prevent merge requests from accidentally being accepted before they're +completely ready, GitLab blocks the "Accept" button for merge requests that +have been marked as a **Work In Progress**. + +[Learn more about settings a merge request as "Work In Progress".](work_in_progress_merge_requests.md) + +## Ignore whitespace changes in Merge Request diff view + +If you click the **Hide whitespace changes** button, you can see the diff +without whitespace changes (if there are any). This is also working when on a +specific commit page. + +![MR diff](img/merge_request_diff.png) + +>**Tip:** +You can append `?w=1` while on the diffs page of a merge request to ignore any +whitespace changes. + +## Live preview with Review Apps + +If you configured [Review Apps](https://about.gitlab.com/features/review-apps/) for your project, +you can preview the changes submitted to a feature-branch through a merge request +in a per-branch basis. No need to checkout the branch, install and preview locally; +all your changes will be available to preview by anyone with the Review Apps link. + +[Read more about Review Apps.](../../../ci/review_apps/index.md) + + +## Tips + +Here are some tips that will help you be more efficient with merge requests in +the command line. + +> **Note:** +This section might move in its own document in the future. + +### Checkout merge requests locally + +A merge request contains all the history from a repository, plus the additional +commits added to the branch associated with the merge request. Here's a few +tricks to checkout a merge request locally. + +Please note that you can checkout a merge request locally even if the source +project is a fork (even a private fork) of the target project. + +#### Checkout locally by adding a git alias + +Add the following alias to your `~/.gitconfig`: + +``` +[alias] + mr = !sh -c 'git fetch $1 merge-requests/$2/head:mr-$1-$2 && git checkout mr-$1-$2' - +``` + +Now you can check out a particular merge request from any repository and any +remote. For example, to check out the merge request with ID 5 as shown in GitLab +from the `upstream` remote, do: + +``` +git mr upstream 5 +``` + +This will fetch the merge request into a local `mr-upstream-5` branch and check +it out. + +#### Checkout locally by modifying `.git/config` for a given repository + +Locate the section for your GitLab remote in the `.git/config` file. It looks +like this: + +``` +[remote "origin"] + url = https://gitlab.com/gitlab-org/gitlab-ce.git + fetch = +refs/heads/*:refs/remotes/origin/* +``` + +You can open the file with: + +``` +git config -e +``` + +Now add the following line to the above section: + +``` +fetch = +refs/merge-requests/*/head:refs/remotes/origin/merge-requests/* +``` + +In the end, it should look like this: + +``` +[remote "origin"] + url = https://gitlab.com/gitlab-org/gitlab-ce.git + fetch = +refs/heads/*:refs/remotes/origin/* + fetch = +refs/merge-requests/*/head:refs/remotes/origin/merge-requests/* +``` + +Now you can fetch all the merge requests: + +``` +git fetch origin + +... +From https://gitlab.com/gitlab-org/gitlab-ce.git + * [new ref] refs/merge-requests/1/head -> origin/merge-requests/1 + * [new ref] refs/merge-requests/2/head -> origin/merge-requests/2 +... +``` + +And to check out a particular merge request: + +``` +git checkout origin/merge-requests/1 +``` + +[protected branches]: ../protected_branches.md +[ee]: https://about.gitlab.com/gitlab-ee/ "GitLab Enterprise Edition" \ No newline at end of file -- cgit v1.2.1 From 13b616e72190518b48cbe6a127b8af92b10b685d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 19 Aug 2017 16:16:04 +0200 Subject: Add a changelog entry for merge request pipeline fix --- .../unreleased/fix-gb-fix-head-pipeline-when-pipeline-has-errors.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelogs/unreleased/fix-gb-fix-head-pipeline-when-pipeline-has-errors.yml diff --git a/changelogs/unreleased/fix-gb-fix-head-pipeline-when-pipeline-has-errors.yml b/changelogs/unreleased/fix-gb-fix-head-pipeline-when-pipeline-has-errors.yml new file mode 100644 index 00000000000..ede8031a501 --- /dev/null +++ b/changelogs/unreleased/fix-gb-fix-head-pipeline-when-pipeline-has-errors.yml @@ -0,0 +1,5 @@ +--- +title: Fix merge request pipeline status when pipeline has errors +merge_request: 13664 +author: +type: fixed -- cgit v1.2.1 From f5e54edb11a9b9258310bcd0d833124735b38851 Mon Sep 17 00:00:00 2001 From: winh Date: Tue, 15 Aug 2017 14:07:56 +0200 Subject: Make project selection dropdown consistent --- app/assets/stylesheets/framework/dropdowns.scss | 2 +- app/assets/stylesheets/framework/selects.scss | 38 +++++++++++++++++++++++++ app/assets/stylesheets/framework/variables.scss | 2 +- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/app/assets/stylesheets/framework/dropdowns.scss b/app/assets/stylesheets/framework/dropdowns.scss index 5e768df972f..5f270e288ae 100644 --- a/app/assets/stylesheets/framework/dropdowns.scss +++ b/app/assets/stylesheets/framework/dropdowns.scss @@ -761,7 +761,7 @@ &:hover, &:active, &:focus { - background-color: $gray-darker; + background-color: $dropdown-item-hover-bg; color: $gl-text-color; } diff --git a/app/assets/stylesheets/framework/selects.scss b/app/assets/stylesheets/framework/selects.scss index 40e654f4838..f7a0b355bf1 100644 --- a/app/assets/stylesheets/framework/selects.scss +++ b/app/assets/stylesheets/framework/selects.scss @@ -264,3 +264,41 @@ .ajax-users-dropdown { min-width: 250px !important; } + +// TODO: change global style +.ajax-project-dropdown { + &.select2-drop { + color: $gl-text-color; + } + + .select2-results { + .select2-no-results, + .select2-searching, + .select2-ajax-error, + .select2-selection-limit { + background: transparent; + } + + .select2-result { + padding: 0 1px; + + .select2-match { + font-weight: bold; + text-decoration: none; + } + + .select2-result-label { + padding: #{$gl-padding / 2} $gl-padding; + } + + &.select2-highlighted { + background-color: transparent !important; + color: $gl-text-color; + + .select2-result-label { + background-color: $dropdown-item-hover-bg; + } + } + } + } +} diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss index 3c109a5a929..225d116e9c7 100644 --- a/app/assets/stylesheets/framework/variables.scss +++ b/app/assets/stylesheets/framework/variables.scss @@ -294,7 +294,7 @@ $dropdown-input-focus-shadow: rgba($dropdown-input-focus-border, .4); $dropdown-loading-bg: rgba(#fff, .6); $dropdown-chevron-size: 10px; $dropdown-toggle-active-border-color: darken($border-color, 14%); - +$dropdown-item-hover-bg: $gray-darker; /* * Filtered Search -- cgit v1.2.1 From f5b733ebdd2c6071968e30ae0b684f82795405df Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Mon, 21 Aug 2017 08:38:09 +0200 Subject: Fix broken links in documentation Also, move the old linting docs to the new location. --- doc/api/README.md | 7 +------ doc/api/lint.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 6 deletions(-) create mode 100644 doc/api/lint.md diff --git a/doc/api/README.md b/doc/api/README.md index 8acb2145f1a..266b5f018d9 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -55,15 +55,10 @@ following locations: - [Tags](tags.md) - [Todos](todos.md) - [Users](users.md) -- [Validate CI configuration](ci/lint.md) +- [Validate CI configuration](lint.md) - [V3 to V4](v3_to_v4.md) - [Version](version.md) -The following documentation is for the [internal CI API](ci/README.md): - -- [Builds](ci/builds.md) -- [Runners](ci/runners.md) - ## Road to GraphQL Going forward, we will start on moving to diff --git a/doc/api/lint.md b/doc/api/lint.md new file mode 100644 index 00000000000..bd5a216a99d --- /dev/null +++ b/doc/api/lint.md @@ -0,0 +1,52 @@ +# Validate the .gitlab-ci.yml (API) + +> [Introduced][ce-5953] in GitLab 8.12. + +Checks if your .gitlab-ci.yml file is valid. + +``` +POST /lint +``` + +| Attribute | Type | Required | Description | +| ---------- | ------- | -------- | -------- | +| `content` | string | yes | the .gitlab-ci.yaml content| + +```bash +curl --header "Content-Type: application/json" https://gitlab.example.com/api/v4/ci/lint --data '{"content": "{ \"image\": \"ruby:2.1\", \"services\": [\"postgres\"], \"before_script\": [\"gem install bundler\", \"bundle install\", \"bundle exec rake db:create\"], \"variables\": {\"DB_NAME\": \"postgres\"}, \"types\": [\"test\", \"deploy\", \"notify\"], \"rspec\": { \"script\": \"rake spec\", \"tags\": [\"ruby\", \"postgres\"], \"only\": [\"branches\"]}}"}' +``` + +Be sure to copy paste the exact contents of `.gitlab-ci.yml` as YAML is very picky about indentation and spaces. + +Example responses: + +* Valid content: + + ```json + { + "status": "valid", + "errors": [] + } + ``` + +* Invalid content: + + ```json + { + "status": "invalid", + "errors": [ + "variables config should be a hash of key value pairs" + ] + } + ``` + +* Without the content attribute: + + ```json + { + "error": "content is missing" + } + ``` + +[ce-5953]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5953 + -- cgit v1.2.1 From 2bfee0706a61b7b1d4d5ea82f6c53ffe07e56065 Mon Sep 17 00:00:00 2001 From: Tim Zallmann Date: Mon, 21 Aug 2017 09:45:46 +0000 Subject: Resolve "User profile activity feed contains broken image links" --- app/helpers/events_helper.rb | 2 +- spec/helpers/events_helper_spec.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/app/helpers/events_helper.rb b/app/helpers/events_helper.rb index 722a65eeb98..c6f98e7e782 100644 --- a/app/helpers/events_helper.rb +++ b/app/helpers/events_helper.rb @@ -176,7 +176,7 @@ module EventsHelper sanitize( text, tags: %w(a img gl-emoji b pre code p span), - attributes: Rails::Html::WhiteListSanitizer.allowed_attributes + ['style', 'data-name', 'data-unicode-version'] + attributes: Rails::Html::WhiteListSanitizer.allowed_attributes + ['style', 'data-src', 'data-name', 'data-unicode-version'] ) end diff --git a/spec/helpers/events_helper_spec.rb b/spec/helpers/events_helper_spec.rb index aa138f25bd3..4b72dbb7964 100644 --- a/spec/helpers/events_helper_spec.rb +++ b/spec/helpers/events_helper_spec.rb @@ -62,6 +62,12 @@ describe EventsHelper do expect(helper.event_note(input)).to eq(expected) end + it 'preserves data-src for lazy images' do + input = "![ImageTest](/uploads/test.png)" + image_url = "data-src=\"/uploads/test.png\"" + expect(helper.event_note(input)).to match(image_url) + end + context 'labels formatting' do let(:input) { 'this should be ~label_1' } -- cgit v1.2.1 From 0db5f576fedfa5c4b2a1f9f01a0fdc4cbcd759f9 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Mon, 21 Aug 2017 12:50:09 +0100 Subject: Only require sidekiq-limit_fetch when enabled in settings This gem allows Sidekiq jobs to be throttled. Unfortunately, it has a side-effect: when we haven't enabled job throttling, it will still hit Redis a lot (and miss, because nothing is configured). As this setting already required a restart, ensure that the library is only required when it's enabled. --- Gemfile | 2 +- .../admin/application_settings/_form.html.haml | 4 +- .../unreleased/only-limit-fetch-when-requested.yml | 5 +++ lib/gitlab/sidekiq_throttler.rb | 2 + spec/lib/gitlab/sidekiq_throttler_spec.rb | 50 ++++++++++++++-------- 5 files changed, 44 insertions(+), 19 deletions(-) create mode 100644 changelogs/unreleased/only-limit-fetch-when-requested.yml diff --git a/Gemfile b/Gemfile index 6c8f64bfded..a0a9dddac10 100644 --- a/Gemfile +++ b/Gemfile @@ -152,7 +152,7 @@ gem 'acts-as-taggable-on', '~> 4.0' gem 'sidekiq', '~> 5.0' gem 'sidekiq-cron', '~> 0.6.0' gem 'redis-namespace', '~> 1.5.2' -gem 'sidekiq-limit_fetch', '~> 3.4' +gem 'sidekiq-limit_fetch', '~> 3.4', require: false # Cron Parser gem 'rufus-scheduler', '~> 3.4' diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index 8bf6556079b..dc585054316 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -362,7 +362,9 @@ %fieldset %legend Background Jobs %p - These settings require a restart to take effect. + These settings require a + = link_to 'restart', help_page_path('administration/restart_gitlab') + to take effect. .form-group .col-sm-offset-2.col-sm-10 .checkbox diff --git a/changelogs/unreleased/only-limit-fetch-when-requested.yml b/changelogs/unreleased/only-limit-fetch-when-requested.yml new file mode 100644 index 00000000000..d9acdf56511 --- /dev/null +++ b/changelogs/unreleased/only-limit-fetch-when-requested.yml @@ -0,0 +1,5 @@ +--- +title: Only require Sidekiq throttling library when enabled, to reduce cache misses +merge_request: +author: +type: fixed diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb index d4d39a888e7..5512afa45a8 100644 --- a/lib/gitlab/sidekiq_throttler.rb +++ b/lib/gitlab/sidekiq_throttler.rb @@ -3,6 +3,8 @@ module Gitlab class << self def execute! if Gitlab::CurrentSettings.sidekiq_throttling_enabled? + require 'sidekiq-limit_fetch' + Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue| Sidekiq::Queue[queue].limit = queue_limit end diff --git a/spec/lib/gitlab/sidekiq_throttler_spec.rb b/spec/lib/gitlab/sidekiq_throttler_spec.rb index 6374ac80207..2dbb7bb7c34 100644 --- a/spec/lib/gitlab/sidekiq_throttler_spec.rb +++ b/spec/lib/gitlab/sidekiq_throttler_spec.rb @@ -1,28 +1,44 @@ require 'spec_helper' describe Gitlab::SidekiqThrottler do - before do - Sidekiq.options[:concurrency] = 35 - - stub_application_setting( - sidekiq_throttling_enabled: true, - sidekiq_throttling_factor: 0.1, - sidekiq_throttling_queues: %w[build project_cache] - ) - end - describe '#execute!' do - it 'sets limits on the selected queues' do - described_class.execute! + context 'when job throttling is enabled' do + before do + Sidekiq.options[:concurrency] = 35 + + stub_application_setting( + sidekiq_throttling_enabled: true, + sidekiq_throttling_factor: 0.1, + sidekiq_throttling_queues: %w[build project_cache] + ) + end + + it 'requires sidekiq-limit_fetch' do + expect(described_class).to receive(:require).with('sidekiq-limit_fetch').and_call_original + + described_class.execute! + end + + it 'sets limits on the selected queues' do + described_class.execute! + + expect(Sidekiq::Queue['build'].limit).to eq 4 + expect(Sidekiq::Queue['project_cache'].limit).to eq 4 + end + + it 'does not set limits on other queues' do + described_class.execute! - expect(Sidekiq::Queue['build'].limit).to eq 4 - expect(Sidekiq::Queue['project_cache'].limit).to eq 4 + expect(Sidekiq::Queue['merge'].limit).to be_nil + end end - it 'does not set limits on other queues' do - described_class.execute! + context 'when job throttling is disabled' do + it 'does not require sidekiq-limit_fetch' do + expect(described_class).not_to receive(:require).with('sidekiq-limit_fetch') - expect(Sidekiq::Queue['merge'].limit).to be_nil + described_class.execute! + end end end end -- cgit v1.2.1 From 4bfdf8406f064b9499a1dbb5e4e5abcd0616ffe0 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sun, 20 Aug 2017 14:18:42 -0700 Subject: Fix Error 500s when viewing user or group Atom feeds --- app/models/event.rb | 10 ++++++++++ spec/features/atom/users_spec.rb | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/app/models/event.rb b/app/models/event.rb index f2a560a6b56..15ee170ca75 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -83,6 +83,10 @@ class Event < ActiveRecord::Base self.inheritance_column = 'action' class << self + def model_name + ActiveModel::Name.new(self, nil, 'event') + end + def find_sti_class(action) if action.to_i == PUSHED PushEvent @@ -438,6 +442,12 @@ class Event < ActiveRecord::Base EventForMigration.create!(new_attributes) end + def to_partial_path + # We are intentionally using `Event` rather than `self.class` so that + # subclasses also use the `Event` implementation. + Event._to_partial_path + end + private def recent_update? diff --git a/spec/features/atom/users_spec.rb b/spec/features/atom/users_spec.rb index 79069bbca8e..9ce687afb31 100644 --- a/spec/features/atom/users_spec.rb +++ b/spec/features/atom/users_spec.rb @@ -41,6 +41,8 @@ describe "User Feed" do target_project: project, description: "Here is the fix: ![an image](image.png)") end + let(:push_event) { create(:push_event, project: project, author: user) } + let!(:push_event_payload) { create(:push_event_payload, event: push_event) } before do project.team << [user, :master] @@ -70,6 +72,10 @@ describe "User Feed" do it 'has XHTML summaries in merge request descriptions' do expect(body).to match /Here is the fix: ]*>]*\/><\/a>/ end + + it 'has push event commit ID' do + expect(body).to have_content(Commit.truncate_sha(push_event.commit_id)) + end end end -- cgit v1.2.1 From c26bc5097fbb4720f021c9889ba500151bb5b85a Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Mon, 21 Aug 2017 14:30:37 +0100 Subject: Make RuboCop happy --- app/services/merge_requests/create_from_issue_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/merge_requests/create_from_issue_service.rb b/app/services/merge_requests/create_from_issue_service.rb index e9e4513e0d0..da39a380451 100644 --- a/app/services/merge_requests/create_from_issue_service.rb +++ b/app/services/merge_requests/create_from_issue_service.rb @@ -4,7 +4,7 @@ module MergeRequests return error('Invalid issue iid') unless issue_iid.present? && issue.present? params[:label_ids] = issue.label_ids if issue.label_ids.any? - + result = CreateBranchService.new(project, current_user).execute(branch_name, ref) return result if result[:status] == :error -- cgit v1.2.1 From f83392e17cc752422658326212735b73d18742e5 Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Mon, 21 Aug 2017 14:08:43 +0000 Subject: Issue tracker --- doc/user/project/issues/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index 759744881f1..05d2b315c66 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -1,6 +1,6 @@ # Issues -GitLab issues is an advanced and complete tool +The GitLab Issue Tracker is an advanced and complete tool for tracking the evolution of a new idea or the process of solving a problem. -- cgit v1.2.1 From 9b15d8e8eb6627b7e12b646c9382668d8d78e425 Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Mon, 21 Aug 2017 14:19:48 +0000 Subject: Cross link to groups docs --- doc/user/group/index.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 9e168e830e5..2ac766e7714 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -55,6 +55,13 @@ By doing so: - John mentions everyone from his team with `@john-team` - John mentions only his marketing team with `@john-team/marketing` +## Group-level views of issues and merge requests + +Issues and merge requests are part of projects. GitLab allows you to view +all issues and merge requests together, within all the projects in particular group, +in dedicated group-level views. These are the [group-level issue list view](../project/issues/index.md#issue-list-views-searching-and-filtering) +and the [group-level merge request list view](../project/merge_requests/index.md#merge-request-list-views-searching-and-filtering). + ## Create a new group > **Notes:** -- cgit v1.2.1 From 71b89a87442f7a03387376fff60df8ba2233260b Mon Sep 17 00:00:00 2001 From: Victor Wu Date: Mon, 21 Aug 2017 14:25:07 +0000 Subject: Wording --- doc/user/group/index.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 2ac766e7714..a89acd907b7 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -55,12 +55,12 @@ By doing so: - John mentions everyone from his team with `@john-team` - John mentions only his marketing team with `@john-team/marketing` -## Group-level views of issues and merge requests +## Group-level list views of issues and merge requests -Issues and merge requests are part of projects. GitLab allows you to view -all issues and merge requests together, within all the projects in particular group, -in dedicated group-level views. These are the [group-level issue list view](../project/issues/index.md#issue-list-views-searching-and-filtering) -and the [group-level merge request list view](../project/merge_requests/index.md#merge-request-list-views-searching-and-filtering). +Issues and merge requests are part of projects. For a given group, view all the +issues across all the projects in that group, together in a single list view. This +is the [group-level issue list view](../project/issues/index.md#issue-list-views-searching-and-filtering). +Similarly, there is a [group-level merge request list view](../project/merge_requests/index.md#merge-request-list-views-searching-and-filtering). ## Create a new group -- cgit v1.2.1 From 5bc9dedf401a10388b9505ccbc11d4802ff76a43 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Wed, 9 Aug 2017 11:58:00 +0100 Subject: Improves subgroup creation permissions --- app/controllers/groups_controller.rb | 7 +++++ app/policies/group_policy.rb | 4 ++- app/services/groups/create_service.rb | 4 +-- app/views/shared/_group_form.html.haml | 5 ++-- ...35845-improve-subgroup-creation-permissions.yml | 5 ++++ spec/features/groups_spec.rb | 17 ++++++++---- spec/policies/group_policy_spec.rb | 30 ++++++++++++++++++++++ spec/services/groups/create_service_spec.rb | 14 +++++++++- 8 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 changelogs/unreleased/35845-improve-subgroup-creation-permissions.yml diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb index f76b3f69e9e..994e736d66e 100644 --- a/app/controllers/groups_controller.rb +++ b/app/controllers/groups_controller.rb @@ -26,6 +26,13 @@ class GroupsController < Groups::ApplicationController def new @group = Group.new + + if params[:parent_id].present? + parent = Group.find_by(id: params[:parent_id]) + if can?(current_user, :create_subgroup, parent) + @group.parent = parent + end + end end def create diff --git a/app/policies/group_policy.rb b/app/policies/group_policy.rb index 6defab75fce..8ada661e571 100644 --- a/app/policies/group_policy.rb +++ b/app/policies/group_policy.rb @@ -13,6 +13,8 @@ class GroupPolicy < BasePolicy condition(:master) { access_level >= GroupMember::MASTER } condition(:reporter) { access_level >= GroupMember::REPORTER } + condition(:nested_groups_supported, scope: :global) { Group.supports_nested_groups? } + condition(:has_projects) do GroupProjectsFinder.new(group: @subject, current_user: @user).execute.any? end @@ -42,7 +44,7 @@ class GroupPolicy < BasePolicy enable :change_visibility_level end - rule { owner & can_create_group }.enable :create_subgroup + rule { owner & can_create_group & nested_groups_supported }.enable :create_subgroup rule { public_group | logged_in_viewable }.enable :view_globally diff --git a/app/services/groups/create_service.rb b/app/services/groups/create_service.rb index c4e9b8fd8e0..c7c27621085 100644 --- a/app/services/groups/create_service.rb +++ b/app/services/groups/create_service.rb @@ -13,9 +13,9 @@ module Groups return @group end - if @group.parent && !can?(current_user, :admin_group, @group.parent) + if @group.parent && !can?(current_user, :create_subgroup, @group.parent) @group.parent = nil - @group.errors.add(:parent_id, 'manage access required to create subgroup') + @group.errors.add(:parent_id, 'You don’t have permission to create a subgroup in this group.') return @group end diff --git a/app/views/shared/_group_form.html.haml b/app/views/shared/_group_form.html.haml index 8d5b5129454..2e1bd5a088c 100644 --- a/app/views/shared/_group_form.html.haml +++ b/app/views/shared/_group_form.html.haml @@ -1,6 +1,6 @@ - content_for :page_specific_javascripts do = page_specific_javascript_bundle_tag('group') -- parent = GroupFinder.new(current_user).execute(id: params[:parent_id] || @group.parent_id) +- parent = @group.parent - group_path = root_url - group_path << parent.full_path + '/' if parent @@ -13,13 +13,12 @@ %span>= root_url - if parent %strong= parent.full_path + '/' + = f.hidden_field :parent_id = f.text_field :path, placeholder: 'open-source', class: 'form-control', autofocus: local_assigns[:autofocus] || false, required: true, pattern: Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX_JS, title: 'Please choose a group path with no special characters.', "data-bind-in" => "#{'create_chat_team' if Gitlab.config.mattermost.enabled}" - - if parent - = f.hidden_field :parent_id, value: parent.id - if @group.persisted? .alert.alert-warning.prepend-top-10 diff --git a/changelogs/unreleased/35845-improve-subgroup-creation-permissions.yml b/changelogs/unreleased/35845-improve-subgroup-creation-permissions.yml new file mode 100644 index 00000000000..eac8dbe23c2 --- /dev/null +++ b/changelogs/unreleased/35845-improve-subgroup-creation-permissions.yml @@ -0,0 +1,5 @@ +--- +title: Improves subgroup creation permissions +merge_request: 13418 +author: +type: bugifx diff --git a/spec/features/groups_spec.rb b/spec/features/groups_spec.rb index e59a484d992..20f9818b08b 100644 --- a/spec/features/groups_spec.rb +++ b/spec/features/groups_spec.rb @@ -104,18 +104,15 @@ feature 'Group' do end context 'as group owner' do - let(:user) { create(:user) } + it 'creates a nested group' do + user = create(:user) - before do group.add_owner(user) sign_out(:user) sign_in(user) visit subgroups_group_path(group) click_link 'New Subgroup' - end - - it 'creates a nested group' do fill_in 'Group path', with: 'bar' click_button 'Create group' @@ -123,6 +120,16 @@ feature 'Group' do expect(page).to have_content("Group 'bar' was successfully created.") end end + + context 'when nested group feature is disabled' do + it 'renders 404' do + allow(Group).to receive(:supports_nested_groups?).and_return(false) + + visit subgroups_group_path(group) + + expect(page.status_code).to eq(404) + end + end end it 'checks permissions to avoid exposing groups by parent_id' do diff --git a/spec/policies/group_policy_spec.rb b/spec/policies/group_policy_spec.rb index b17a93e3fbe..cf420ae3ea6 100644 --- a/spec/policies/group_policy_spec.rb +++ b/spec/policies/group_policy_spec.rb @@ -123,6 +123,36 @@ describe GroupPolicy do end end + describe 'when nested group support feature is disabled' do + before do + allow(Group).to receive(:supports_nested_groups?).and_return(false) + end + + context 'admin' do + let(:current_user) { admin } + + it 'allows every owner permission except creating subgroups' do + create_subgroup_permission = [:create_subgroup] + updated_owner_permissions = owner_permissions - create_subgroup_permission + + expect_disallowed(*create_subgroup_permission) + expect_allowed(*updated_owner_permissions) + end + end + + context 'owner' do + let(:current_user) { owner } + + it 'allows every owner permission except creating subgroups' do + create_subgroup_permission = [:create_subgroup] + updated_owner_permissions = owner_permissions - create_subgroup_permission + + expect_disallowed(*create_subgroup_permission) + expect_allowed(*updated_owner_permissions) + end + end + end + describe 'private nested group use the highest access level from the group and inherited permissions', :nested_groups do let(:nested_group) { create(:group, :private, parent: group) } diff --git a/spec/services/groups/create_service_spec.rb b/spec/services/groups/create_service_spec.rb index b2175717a70..6973e7ff990 100644 --- a/spec/services/groups/create_service_spec.rb +++ b/spec/services/groups/create_service_spec.rb @@ -32,12 +32,24 @@ describe Groups::CreateService, '#execute' do end it { is_expected.to be_persisted } + + context 'when nested groups feature is disabled' do + it 'does not save group and returns an error' do + allow(Group).to receive(:supports_nested_groups?).and_return(false) + + is_expected.not_to be_persisted + expect(subject.errors[:parent_id]).to include('You don’t have permission to create a subgroup in this group.') + expect(subject.parent_id).to be_nil + end + end end context 'as guest' do it 'does not save group and returns an error' do + allow(Group).to receive(:supports_nested_groups?).and_return(true) + is_expected.not_to be_persisted - expect(subject.errors[:parent_id].first).to eq('manage access required to create subgroup') + expect(subject.errors[:parent_id].first).to eq('You don’t have permission to create a subgroup in this group.') expect(subject.parent_id).to be_nil end end -- cgit v1.2.1 From b7ebb447d7a0f8e65c791d4f7af6e042b72ff644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Mon, 21 Aug 2017 15:13:40 -0300 Subject: Correctly encode string params for Gitaly's TreeEntries RPC --- lib/gitlab/gitaly_client/commit_service.rb | 4 ++-- spec/lib/gitlab/gitaly_client/commit_service_spec.rb | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index b36e81278d6..2d58fb0186e 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -80,8 +80,8 @@ module Gitlab def tree_entries(repository, revision, path) request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, - revision: revision, - path: path.presence || '.' + revision: GitalyClient.encode(revision), + path: path.present? ? GitalyClient.encode(path) : '.' ) response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request) diff --git a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb index 7fe698fcb18..2eaf4222964 100644 --- a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb @@ -111,6 +111,20 @@ describe Gitlab::GitalyClient::CommitService do client.tree_entries(repository, revision, path) end + + context 'with UTF-8 params strings' do + let(:revision) { "branch\u011F" } + let(:path) { "foo/\u011F.txt" } + + it 'handles string encodings correctly' do + expect_any_instance_of(Gitaly::CommitService::Stub) + .to receive(:get_tree_entries) + .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) + .and_return([]) + + client.tree_entries(repository, revision, path) + end + end end describe '#find_commit' do -- cgit v1.2.1 From 905caab81a95d89f8426b8bbe530d843dd718354 Mon Sep 17 00:00:00 2001 From: Marcia Ramos Date: Mon, 21 Aug 2017 15:34:52 -0300 Subject: indexes user/search/ from /user/index --- doc/user/index.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/user/index.md b/doc/user/index.md index d664fd62754..e9ec603f2f1 100644 --- a/doc/user/index.md +++ b/doc/user/index.md @@ -126,6 +126,10 @@ are a tool for working faster and more effectively with your team, by listing all user or group mentions, as well as issues and merge requests you're assigned to. +## Search + +[Search and filter](search/index.md) through groups, projects, issues, merge requests, files, code, and more. + ## Snippets [Snippets](snippets.md) are code blocks that you want to store in GitLab, from which -- cgit v1.2.1 From 2eb07aecc93973983c4fc1ce1820b6155e3175ce Mon Sep 17 00:00:00 2001 From: Marcia Ramos Date: Mon, 21 Aug 2017 16:04:52 -0300 Subject: copyedit --- doc/user/group/index.md | 7 +++---- doc/user/project/issues/index.md | 21 ++++++++++++++------- doc/user/project/merge_requests/index.md | 14 ++++++-------- doc/user/search/index.md | 4 ++-- 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/doc/user/group/index.md b/doc/user/group/index.md index a89acd907b7..fbc05261a32 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -55,12 +55,11 @@ By doing so: - John mentions everyone from his team with `@john-team` - John mentions only his marketing team with `@john-team/marketing` -## Group-level list views of issues and merge requests +## Issues and merge requests within a group Issues and merge requests are part of projects. For a given group, view all the -issues across all the projects in that group, together in a single list view. This -is the [group-level issue list view](../project/issues/index.md#issue-list-views-searching-and-filtering). -Similarly, there is a [group-level merge request list view](../project/merge_requests/index.md#merge-request-list-views-searching-and-filtering). +[issues](../project/issues/index.md#issues-per-group) and [merge requests](../project/merge_requests/index.md#merge-requests-per-group) across all the projects in that group, +together in a single list view. ## Create a new group diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index 05d2b315c66..67f0230b36d 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -7,7 +7,7 @@ of solving a problem. It allows you, your team, and your collaborators to share and discuss proposals before and while implementing them. -GitLab issues is available in all +GitLab Issues and the GitLab Issue Tracker are available in all [GitLab Products](https://about.gitlab.com/products/) as part of the [GitLab Workflow](https://about.gitlab.com/2016/10/25/gitlab-workflow-an-overview/). @@ -46,14 +46,23 @@ Create [issue templates](#issue-templates) to make collaboration consistent and containing all information you need. For example, you can create a template for feature proposals and another one for bug reports. -## Issue list views, searching, and filtering +## Issue Tracker -View all the issues within a project by navigating to **Project > Issues**. -There are tabs to quickly filter by open and closed issues. +The Issue Tracker is the collection of opened and closed issues created in a project. +It is available for all projects, from the moment the project is created. + +Find the issue tracker by navigating to your **Project's homepage** > **Issues**. + +### Issues per project + +When you access your project's issues, GitLab will present them in a list, +and you can use the tabs available to quickly filter by open and closed issues. ![Project issues list view](img/project_issues_list_view.png) -[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-project). +You can also [search and filter](../../search/index.md#issues-and-merge-requests-per-project) the results more deeply with GitLab's search capacities. + +### Issues per group View all the issues in a group (that is, all the issues across all projects in that group) by navigating to **Group > Issues**. This view also has the open and closed @@ -61,8 +70,6 @@ issue tabs. ![Group Issues list view](img/group_issues_list_view.png) -[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-group). - ## GitLab Issues Functionalities The image bellow illustrates how an issue looks like: diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index cd52471922f..285c40729fe 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -56,24 +56,23 @@ B. Consider you're a web developer writing a webpage for your company's: 1. Once approved, your merge request is [squashed and merged](https://docs.gitlab.com/ee/user/project/merge_requests/squash_and_merge.html), and [deployed to staging with GitLab Pages](https://about.gitlab.com/2016/08/26/ci-deployment-and-environments/) (Squash and Merge is available in GitLab Enterprise Edition Starter) 1. Your production team [cherry picks](#cherry-pick-changes) the merge commit into production +## Merge requests per project -## Merge request list views, searching, and filtering +View all the merge requests within a project by navigating to **Project > Merge Requests**. -View all the merge requests within a project by navigating to **Project > Merge Requests**. -There are tabs to quickly filter by open, merged, and closed merge requests. +When you access your project's merge requests, GitLab will present them in a list, +and you can use the tabs available to quickly filter by open and closed. You can also [search and filter the results](../../search/index.md#issues-and-merge-requests-per-project). ![Project merge requests list view](img/project_merge_requests_list_view.png) -[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-project). +## Merge requests per group View all the merge requests in a group (that is, all the merge requests across all projects in that group) by navigating to **Group > Merge Requests**. This view also has the open, merged, and closed -merge request tabs. +merge request tabs, from which you can [search and filter the results](../../search/index.md#issues-and-merge-requests-per-group). ![Group Issues list view](img/group_merge_requests_list_view.png) -[Search and filter the results](../../search/index.md#issues-and-merge-requests-per-group). - ## Authorization for merge requests There are two main ways to have a merge request flow with GitLab: @@ -159,7 +158,6 @@ all your changes will be available to preview by anyone with the Review Apps lin [Read more about Review Apps.](../../../ci/review_apps/index.md) - ## Tips Here are some tips that will help you be more efficient with merge requests in diff --git a/doc/user/search/index.md b/doc/user/search/index.md index 52af864d54e..f5c7ce49e8e 100644 --- a/doc/user/search/index.md +++ b/doc/user/search/index.md @@ -27,7 +27,7 @@ on the search field on the top-right of your screen: ![shortcut to your issues and mrs](img/issues_mrs_shortcut.png) -## Issues and merge requests per project +### Issues and merge requests per project If you want to search for issues present in a specific project, navigate to a project's **Issues** tab, and click on the field **Search or filter results...**. It will @@ -40,7 +40,7 @@ The same process is valid for merge requests. Navigate to your project's **Merge and click **Search or filter results...**. Merge requests can be filtered by author, assignee, milestone, and label. -## Issues and merge requests per group +### Issues and merge requests per group Similar to **Issues and merge requests per project**, you can also search for issues within a group. Navigate to a group's **Issues** tab and query search results in -- cgit v1.2.1 From 4b142701cc08d21d80dadca27a15645573a7316a Mon Sep 17 00:00:00 2001 From: Marcia Ramos Date: Mon, 21 Aug 2017 16:05:11 -0300 Subject: indexes external issue tracker --- doc/user/project/issues/index.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index 67f0230b36d..20901e01f6e 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -136,6 +136,12 @@ to find out more about this feature. With [GitLab Enterprise Edition Starter](https://about.gitlab.com/gitlab-ee/), you can also create various boards per project with [Multiple Issue Boards](https://docs.gitlab.com/ee/user/project/issue_board.html#multiple-issue-boards). +### External Issue Tracker + +Alternatively to GitLab's built-in Issue Tracker, you can also use an [external +tracker](../../../integration/external-issue-tracker.md) such as Jira, Redmine, +or Bugzilla. + ### Issue's API Read through the [API documentation](../../../api/issues.md). -- cgit v1.2.1 From 4add91b6e15b242b28b797c8c09bc180c3aec5bc Mon Sep 17 00:00:00 2001 From: Bryce Johnson Date: Mon, 21 Aug 2017 21:29:09 +0000 Subject: Use non-i18n values for setting new group-level issue/MR button text --- .../javascripts/project_select_combo_button.js | 33 ++++++++++++-------- app/views/dashboard/issues.html.haml | 4 +-- app/views/dashboard/merge_requests.html.haml | 4 +-- app/views/dashboard/milestones/index.html.haml | 4 +-- app/views/groups/issues.html.haml | 4 +-- app/views/groups/merge_requests.html.haml | 4 +-- .../shared/_new_project_item_select.html.haml | 2 +- app/views/shared/empty_states/_issues.html.haml | 2 +- .../shared/empty_states/_merge_requests.html.haml | 2 +- .../fixtures/project_select_combo_button.html.haml | 2 +- .../project_select_combo_button_spec.js | 35 ++++++++++++++++++++++ 11 files changed, 69 insertions(+), 27 deletions(-) diff --git a/app/assets/javascripts/project_select_combo_button.js b/app/assets/javascripts/project_select_combo_button.js index f799d9d619a..46a26fb91f4 100644 --- a/app/assets/javascripts/project_select_combo_button.js +++ b/app/assets/javascripts/project_select_combo_button.js @@ -4,10 +4,10 @@ export default class ProjectSelectComboButton { constructor(select) { this.projectSelectInput = $(select); this.newItemBtn = $('.new-project-item-link'); - this.newItemBtnBaseText = this.newItemBtn.data('label'); - this.itemType = this.deriveItemTypeFromLabel(); + this.resourceType = this.newItemBtn.data('type'); + this.resourceLabel = this.newItemBtn.data('label'); + this.formattedText = this.deriveTextVariants(); this.groupId = this.projectSelectInput.data('groupId'); - this.bindEvents(); this.initLocalStorage(); } @@ -23,9 +23,7 @@ export default class ProjectSelectComboButton { const localStorageIsSafe = AccessorUtilities.isLocalStorageAccessSafe(); if (localStorageIsSafe) { - const itemTypeKebabed = this.newItemBtnBaseText.toLowerCase().split(' ').join('-'); - - this.localStorageKey = ['group', this.groupId, itemTypeKebabed, 'recent-project'].join('-'); + this.localStorageKey = ['group', this.groupId, this.formattedText.localStorageItemType, 'recent-project'].join('-'); this.setBtnTextFromLocalStorage(); } } @@ -57,19 +55,14 @@ export default class ProjectSelectComboButton { setNewItemBtnAttributes(project) { if (project) { this.newItemBtn.attr('href', project.url); - this.newItemBtn.text(`${this.newItemBtnBaseText} in ${project.name}`); + this.newItemBtn.text(`${this.formattedText.defaultTextPrefix} in ${project.name}`); this.newItemBtn.enable(); } else { - this.newItemBtn.text(`Select project to create ${this.itemType}`); + this.newItemBtn.text(`Select project to create ${this.formattedText.presetTextSuffix}`); this.newItemBtn.disable(); } } - deriveItemTypeFromLabel() { - // label is either 'New issue' or 'New merge request' - return this.newItemBtnBaseText.split(' ').slice(1).join(' '); - } - getProjectFromLocalStorage() { const projectString = localStorage.getItem(this.localStorageKey); @@ -81,5 +74,19 @@ export default class ProjectSelectComboButton { localStorage.setItem(this.localStorageKey, projectString); } + + deriveTextVariants() { + const defaultTextPrefix = this.resourceLabel; + + // the trailing slice call depluralizes each of these strings (e.g. new-issues -> new-issue) + const localStorageItemType = `new-${this.resourceType.split('_').join('-').slice(0, -1)}`; + const presetTextSuffix = this.resourceType.split('_').join(' ').slice(0, -1); + + return { + localStorageItemType, // new-issue / new-merge-request + defaultTextPrefix, // New issue / New merge request + presetTextSuffix, // issue / merge request + }; + } } diff --git a/app/views/dashboard/issues.html.haml b/app/views/dashboard/issues.html.haml index 52e0012fd7d..9ac44674b73 100644 --- a/app/views/dashboard/issues.html.haml +++ b/app/views/dashboard/issues.html.haml @@ -8,14 +8,14 @@ - content_for :breadcrumbs_extra do = link_to params.merge(rss_url_options), class: 'btn has-tooltip append-right-10', title: 'Subscribe' do = icon('rss') - = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", with_feature_enabled: 'issues' + = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", with_feature_enabled: 'issues', type: :issues .top-area = render 'shared/issuable/nav', type: :issues .nav-controls{ class: ("visible-xs" if show_new_nav?) } = link_to params.merge(rss_url_options), class: 'btn has-tooltip', title: 'Subscribe' do = icon('rss') - = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", with_feature_enabled: 'issues' + = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", with_feature_enabled: 'issues', type: :issues = render 'shared/issuable/filter', type: :issues = render 'shared/issues' diff --git a/app/views/dashboard/merge_requests.html.haml b/app/views/dashboard/merge_requests.html.haml index c3fe14da2b2..960e1e55f36 100644 --- a/app/views/dashboard/merge_requests.html.haml +++ b/app/views/dashboard/merge_requests.html.haml @@ -4,12 +4,12 @@ - if show_new_nav? - content_for :breadcrumbs_extra do - = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", with_feature_enabled: 'merge_requests' + = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", with_feature_enabled: 'merge_requests', type: :merge_requests .top-area = render 'shared/issuable/nav', type: :merge_requests .nav-controls{ class: ("visible-xs" if show_new_nav?) } - = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", with_feature_enabled: 'merge_requests' + = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", with_feature_enabled: 'merge_requests', type: :merge_requests = render 'shared/issuable/filter', type: :merge_requests = render 'shared/merge_requests' diff --git a/app/views/dashboard/milestones/index.html.haml b/app/views/dashboard/milestones/index.html.haml index 37dbcaf5cb8..cb8bf57cba1 100644 --- a/app/views/dashboard/milestones/index.html.haml +++ b/app/views/dashboard/milestones/index.html.haml @@ -4,13 +4,13 @@ - if show_new_nav? - content_for :breadcrumbs_extra do - = render 'shared/new_project_item_select', path: 'milestones/new', label: 'New milestone', include_groups: true + = render 'shared/new_project_item_select', path: 'milestones/new', label: 'New milestone', include_groups: true, type: :milestones .top-area = render 'shared/milestones_filter', counts: @milestone_states .nav-controls{ class: ("visible-xs" if show_new_nav?) } - = render 'shared/new_project_item_select', path: 'milestones/new', label: 'New milestone', include_groups: true + = render 'shared/new_project_item_select', path: 'milestones/new', label: 'New milestone', include_groups: true, type: :milestones .milestones %ul.content-list diff --git a/app/views/groups/issues.html.haml b/app/views/groups/issues.html.haml index f83ebbf09ef..12bc092d216 100644 --- a/app/views/groups/issues.html.haml +++ b/app/views/groups/issues.html.haml @@ -12,7 +12,7 @@ - content_for :breadcrumbs_extra do = link_to params.merge(rss_url_options), class: 'btn btn-default append-right-10' do = icon('rss') - = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue" + = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", type: :issues - if group_issues_exists .top-area @@ -22,7 +22,7 @@ = icon('rss') %span.icon-label Subscribe - = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue" + = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", type: :issues = render 'shared/issuable/search_bar', type: :issues diff --git a/app/views/groups/merge_requests.html.haml b/app/views/groups/merge_requests.html.haml index 997c82c77d9..569eef46e6e 100644 --- a/app/views/groups/merge_requests.html.haml +++ b/app/views/groups/merge_requests.html.haml @@ -2,7 +2,7 @@ - if show_new_nav? && current_user - content_for :breadcrumbs_extra do - = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request" + = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", type: :merge_requests - if @group_merge_requests.empty? = render 'shared/empty_states/merge_requests', project_select_button: true @@ -11,7 +11,7 @@ = render 'shared/issuable/nav', type: :merge_requests - if current_user .nav-controls{ class: ("visible-xs" if show_new_nav?) } - = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request" + = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", type: :merge_requests = render 'shared/issuable/filter', type: :merge_requests diff --git a/app/views/shared/_new_project_item_select.html.haml b/app/views/shared/_new_project_item_select.html.haml index 96502d7ce93..dc912d800cf 100644 --- a/app/views/shared/_new_project_item_select.html.haml +++ b/app/views/shared/_new_project_item_select.html.haml @@ -1,6 +1,6 @@ - if any_projects?(@projects) .project-item-select-holder.btn-group.pull-right - %a.btn.btn-new.new-project-item-link{ href: '', data: { label: local_assigns[:label] } } + %a.btn.btn-new.new-project-item-link{ href: '', data: { label: local_assigns[:label], type: local_assigns[:type] } } = icon('spinner spin') = project_select_tag :project_path, class: "project-item-select", data: { include_groups: local_assigns[:include_groups], order_by: 'last_activity_at', relative_path: local_assigns[:path] }, with_feature_enabled: local_assigns[:with_feature_enabled] %button.btn.btn-new.new-project-item-select-button diff --git a/app/views/shared/empty_states/_issues.html.haml b/app/views/shared/empty_states/_issues.html.haml index b0c0ab523c7..68737e8da66 100644 --- a/app/views/shared/empty_states/_issues.html.haml +++ b/app/views/shared/empty_states/_issues.html.haml @@ -15,7 +15,7 @@ Issues can be bugs, tasks or ideas to be discussed. Also, issues are searchable and filterable. - if project_select_button - = render 'shared/new_project_item_select', path: 'issues/new', label: 'New issue' + = render 'shared/new_project_item_select', path: 'issues/new', label: 'New issue', type: :issues - else = link_to 'New issue', button_path, class: 'btn btn-new', title: 'New issue', id: 'new_issue_link' - else diff --git a/app/views/shared/empty_states/_merge_requests.html.haml b/app/views/shared/empty_states/_merge_requests.html.haml index 3e64f403b8b..ff5741b6d61 100644 --- a/app/views/shared/empty_states/_merge_requests.html.haml +++ b/app/views/shared/empty_states/_merge_requests.html.haml @@ -14,7 +14,7 @@ %p Interested parties can even contribute by pushing commits if they want to. - if project_select_button - = render 'shared/new_project_item_select', path: 'merge_requests/new', label: 'New merge request' + = render 'shared/new_project_item_select', path: 'merge_requests/new', label: 'New merge request', type: :merge_requests - else = link_to 'New merge request', button_path, class: 'btn btn-new', title: 'New merge request', id: 'new_merge_request_link' - else diff --git a/spec/javascripts/fixtures/project_select_combo_button.html.haml b/spec/javascripts/fixtures/project_select_combo_button.html.haml index 54bc1a59279..432cd5fcc74 100644 --- a/spec/javascripts/fixtures/project_select_combo_button.html.haml +++ b/spec/javascripts/fixtures/project_select_combo_button.html.haml @@ -1,6 +1,6 @@ .project-item-select-holder %input.project-item-select{ data: { group_id: '12345' , relative_path: 'issues/new' } } - %a.new-project-item-link{ data: { label: 'New issue' }, href: ''} + %a.new-project-item-link{ data: { label: 'New issue', type: 'issues' }, href: ''} %i.fa.fa-spinner.spin %a.new-project-item-select-button %i.fa.fa-caret-down diff --git a/spec/javascripts/project_select_combo_button_spec.js b/spec/javascripts/project_select_combo_button_spec.js index e10a5a3bef6..021804e0769 100644 --- a/spec/javascripts/project_select_combo_button_spec.js +++ b/spec/javascripts/project_select_combo_button_spec.js @@ -101,5 +101,40 @@ describe('Project Select Combo Button', function () { window.localStorage.clear(); }); }); + + describe('deriveTextVariants', function () { + beforeEach(function () { + this.mockExecutionContext = { + resourceType: '', + resourceLabel: '', + }; + + this.comboButton = new ProjectSelectComboButton(this.projectSelectInput); + + this.method = this.comboButton.deriveTextVariants.bind(this.mockExecutionContext); + }); + + it('correctly derives test variants for merge requests', function () { + this.mockExecutionContext.resourceType = 'merge_requests'; + this.mockExecutionContext.resourceLabel = 'New merge request'; + + const returnedVariants = this.method(); + + expect(returnedVariants.localStorageItemType).toBe('new-merge-request'); + expect(returnedVariants.defaultTextPrefix).toBe('New merge request'); + expect(returnedVariants.presetTextSuffix).toBe('merge request'); + }); + + it('correctly derives text variants for issues', function () { + this.mockExecutionContext.resourceType = 'issues'; + this.mockExecutionContext.resourceLabel = 'New issue'; + + const returnedVariants = this.method(); + + expect(returnedVariants.localStorageItemType).toBe('new-issue'); + expect(returnedVariants.defaultTextPrefix).toBe('New issue'); + expect(returnedVariants.presetTextSuffix).toBe('issue'); + }); + }); }); -- cgit v1.2.1 From cdafffcf158977e39161c1ad0516d91cef66ddbc Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 2 Aug 2017 03:05:48 +0200 Subject: Codestyle: move hooks to the same place and move dependent methods to private --- app/models/project.rb | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index be248bc99e1..ba5fca75729 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -44,31 +44,23 @@ class Project < ActiveRecord::Base default_value_for :snippets_enabled, gitlab_config_features.snippets default_value_for :only_allow_merge_if_all_discussions_are_resolved, false - after_create :ensure_storage_path_exist - after_create :create_project_feature, unless: :project_feature - after_save :update_project_statistics, if: :namespace_id_changed? + add_authentication_token_field :runners_token + before_save :ensure_runners_token - # set last_activity_at to the same as created_at + after_save :update_project_statistics, if: :namespace_id_changed? + after_create :create_project_feature, unless: :project_feature after_create :set_last_activity_at - def set_last_activity_at - update_column(:last_activity_at, self.created_at) - end - after_create :set_last_repository_updated_at - def set_last_repository_updated_at - update_column(:last_repository_updated_at, self.created_at) - end + after_update :update_forks_visibility_level before_destroy :remove_private_deploy_keys after_destroy -> { run_after_commit { remove_pages } } - # update visibility_level of forks - after_update :update_forks_visibility_level - after_validation :check_pending_delete - # Legacy Storage specific hooks - + # Storage specific hooks + after_initialize :load_storage + after_create :ensure_storage_path_exist after_save :ensure_storage_path_exist, if: :namespace_id_changed? acts_as_taggable @@ -238,9 +230,6 @@ class Project < ActiveRecord::Base presence: true, inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } - add_authentication_token_field :runners_token - before_save :ensure_runners_token - mount_uploader :avatar, AvatarUploader has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -1086,6 +1075,7 @@ class Project < ActiveRecord::Base !!repository.exists? end + # update visibility_level of forks def update_forks_visibility_level return unless visibility_level < visibility_level_was @@ -1421,6 +1411,15 @@ class Project < ActiveRecord::Base private + # set last_activity_at to the same as created_at + def set_last_activity_at + update_column(:last_activity_at, self.created_at) + end + + def set_last_repository_updated_at + update_column(:last_repository_updated_at, self.created_at) + end + def cross_namespace_reference?(from) case from when Project -- cgit v1.2.1 From 950c87308aa7e9f3ae5bf84790d30d6fcb688fc8 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 2 Aug 2017 03:14:44 +0200 Subject: Move create_repository back to project model as we can use disk_path and share it --- app/models/concerns/storage/legacy_project.rb | 13 ------------- app/models/project.rb | 13 +++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/app/models/concerns/storage/legacy_project.rb b/app/models/concerns/storage/legacy_project.rb index 815db712285..c38042495f4 100644 --- a/app/models/concerns/storage/legacy_project.rb +++ b/app/models/concerns/storage/legacy_project.rb @@ -59,18 +59,5 @@ module Storage Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path) Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path) end - - def create_repository(force: false) - # Forked import is handled asynchronously - return if forked? && !force - - if gitlab_shell.add_repository(repository_storage_path, path_with_namespace) - repository.after_create - true - else - errors.add(:base, 'Failed to create repository via gitlab-shell') - false - end - end end end diff --git a/app/models/project.rb b/app/models/project.rb index ba5fca75729..570955e81dd 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -993,6 +993,19 @@ class Project < ActiveRecord::Base end end + def create_repository(force: false) + # Forked import is handled asynchronously + return if forked? && !force + + if gitlab_shell.add_repository(repository_storage_path, disk_path) + repository.after_create + true + else + errors.add(:base, 'Failed to create repository via gitlab-shell') + false + end + end + def hook_attrs(backward: true) attrs = { name: name, -- cgit v1.2.1 From 53403399577bdca0e8f0886fa62ce0e75c14a8e0 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 2 Aug 2017 04:11:35 +0200 Subject: Add UUID Storage to Project --- app/models/concerns/storage/uuid_project.rb | 58 ++++++++++++++++++++++ app/models/project.rb | 9 +++- ...20170802013652_add_storage_fields_to_project.rb | 39 +++++++++++++++ db/schema.rb | 3 ++ 4 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 app/models/concerns/storage/uuid_project.rb create mode 100644 db/migrate/20170802013652_add_storage_fields_to_project.rb diff --git a/app/models/concerns/storage/uuid_project.rb b/app/models/concerns/storage/uuid_project.rb new file mode 100644 index 00000000000..8a73287e518 --- /dev/null +++ b/app/models/concerns/storage/uuid_project.rb @@ -0,0 +1,58 @@ +module Storage + module UUIDProject + extend ActiveSupport::Concern + + def uuid_dir + %Q(#{uuid[0..1]}/#{uuid[2..3]}) + end + + def disk_path + %Q(#{uuid_dir}/#{uuid}) + end + + def ensure_storage_path_exist + gitlab_shell.add_namespace(repository_storage_path, uuid_dir) + end + + def rename_repo + # TODO: We cannot wipe most of this method until we provide migration path for Container Registries + path_was = previous_changes['path'].first + old_path_with_namespace = File.join(namespace.full_path, path_was) + new_path_with_namespace = File.join(namespace.full_path, path) + + Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}" + + if has_container_registry_tags? + Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" + + # we currently doesn't support renaming repository if it contains images in container registry + raise StandardError.new('Project cannot be renamed, because images are present in its container registry') + end + + begin + # TODO: we can avoid cache expiration if cache is based on UUID or just project_id + expire_caches_before_rename(old_path_with_namespace) + expires_full_path_cache + + send_move_instructions(old_path_with_namespace) + + @old_path_with_namespace = old_path_with_namespace + + SystemHooksService.new.execute_hooks_for(self, :rename) + + @repository = nil + rescue => e + Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" + # Returning false does not rollback after_* transaction but gives + # us information about failing some of tasks + false + end + + Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" + + # TODO: When we move Uploads and Pages to use UUID we can disable this transfers as well + Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path) + Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path) + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 570955e81dd..f0ea83dda0c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -17,7 +17,6 @@ class Project < ActiveRecord::Base include ProjectFeaturesCompatibility include SelectForProjectAuthorization include Routable - include Storage::LegacyProject extend Gitlab::ConfigHelper @@ -1424,6 +1423,14 @@ class Project < ActiveRecord::Base private + def load_storage + if self.storage_version > 1 + self.class.include Storage::UUIDProject + else + self.class.include Storage::LegacyProject + end + end + # set last_activity_at to the same as created_at def set_last_activity_at update_column(:last_activity_at, self.created_at) diff --git a/db/migrate/20170802013652_add_storage_fields_to_project.rb b/db/migrate/20170802013652_add_storage_fields_to_project.rb new file mode 100644 index 00000000000..269103cd472 --- /dev/null +++ b/db/migrate/20170802013652_add_storage_fields_to_project.rb @@ -0,0 +1,39 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddStorageFieldsToProject < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index", "remove_concurrent_index" or + # "add_column_with_default" you must disable the use of transactions + # as these methods can not run in an existing transaction. + # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure + # that either of them is the _only_ method called in the migration, + # any other changes should go in a separate migration. + # This ensures that upon failure _only_ the index creation or removing fails + # and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + disable_ddl_transaction! + + def up + # rubocop:disable Migration/AddColumnWithDefaultToLargeTable + add_column :projects, :uuid, :uuid + add_column_with_default :projects, :storage_version, :integer, default: 0, limit: 1 + add_concurrent_index :projects, :uuid + end + + def down + remove_column :projects, :uuid + remove_column :projects, :storage_version + end +end diff --git a/db/schema.rb b/db/schema.rb index c31bff3a8f2..dcd9532e4be 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1208,6 +1208,8 @@ ActiveRecord::Schema.define(version: 20170820100558) do t.datetime "last_repository_updated_at" t.string "ci_config_path" t.text "delete_error" + t.uuid "uuid" + t.integer "storage_version", limit: 2, default: 0, null: false end add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree @@ -1224,6 +1226,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do add_index "projects", ["pending_delete"], name: "index_projects_on_pending_delete", using: :btree add_index "projects", ["runners_token"], name: "index_projects_on_runners_token", using: :btree add_index "projects", ["star_count"], name: "index_projects_on_star_count", using: :btree + add_index "projects", ["uuid"], name: "index_projects_on_uuid", using: :btree add_index "projects", ["visibility_level"], name: "index_projects_on_visibility_level", using: :btree create_table "protected_branch_merge_access_levels", force: :cascade do |t| -- cgit v1.2.1 From 9e6fa996eab978506af1084b79a9c3f91f6d575b Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Fri, 4 Aug 2017 07:30:42 +0200 Subject: New storage is now "Hashed" instead of "UUID" --- app/models/concerns/storage/hashed_project.rb | 72 +++++++ app/models/concerns/storage/legacy_project.rb | 12 +- app/models/concerns/storage/uuid_project.rb | 58 ------ app/models/project.rb | 8 +- ...20170802013652_add_storage_fields_to_project.rb | 24 +-- db/schema.rb | 5 +- lib/gitlab/import_export/import_export.yml | 1 + spec/factories/projects.rb | 4 + spec/models/project_spec.rb | 213 +++++++++++++++------ 9 files changed, 256 insertions(+), 141 deletions(-) create mode 100644 app/models/concerns/storage/hashed_project.rb delete mode 100644 app/models/concerns/storage/uuid_project.rb diff --git a/app/models/concerns/storage/hashed_project.rb b/app/models/concerns/storage/hashed_project.rb new file mode 100644 index 00000000000..292a73903b5 --- /dev/null +++ b/app/models/concerns/storage/hashed_project.rb @@ -0,0 +1,72 @@ +module Storage + module HashedProject + extend ActiveSupport::Concern + + # Base directory + # + # @return [String] directory where repository is stored + def base_dir + %Q(#{disk_hash[0..1]}/#{disk_hash[2..3]}) if disk_hash + end + + # Disk path is used to build repository and project's wiki path on disk + # + # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions + def disk_path + %Q(#{base_dir}/#{disk_hash}) + end + + def ensure_storage_path_exist + gitlab_shell.add_namespace(repository_storage_path, base_dir) + end + + def rename_repo + # TODO: We cannot wipe most of this method until we provide migration path for Container Registries + path_was = previous_changes['path'].first + old_path_with_namespace = File.join(namespace.full_path, path_was) + new_path_with_namespace = File.join(namespace.full_path, path) + + Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}" + + if has_container_registry_tags? + Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" + + # we currently doesn't support renaming repository if it contains images in container registry + raise StandardError.new('Project cannot be renamed, because images are present in its container registry') + end + + begin + # TODO: we can avoid cache expiration if cache is based on UUID or just project_id + expire_caches_before_rename(old_path_with_namespace) + expires_full_path_cache + + send_move_instructions(old_path_with_namespace) + + @old_path_with_namespace = old_path_with_namespace + + SystemHooksService.new.execute_hooks_for(self, :rename) + + @repository = nil + rescue => e + Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" + # Returning false does not rollback after_* transaction but gives + # us information about failing some of tasks + false + end + + Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" + + # TODO: When we move Uploads and Pages to use UUID we can disable this transfers as well + Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path) + Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path) + end + + private + + # Generates the hash for the project path and name on disk + # If you need to refer to the repository on disk, use the `#disk_path` + def disk_hash + @disk_hash ||= Digest::SHA2.hexdigest(self.id.to_s) if self.id + end + end +end diff --git a/app/models/concerns/storage/legacy_project.rb b/app/models/concerns/storage/legacy_project.rb index c38042495f4..839bbcc76ea 100644 --- a/app/models/concerns/storage/legacy_project.rb +++ b/app/models/concerns/storage/legacy_project.rb @@ -2,12 +2,22 @@ module Storage module LegacyProject extend ActiveSupport::Concern + # Base directory + # + # @return [String] directory where repository is stored + def base_dir + namespace.full_path + end + + # Disk path is used to build repository and project's wiki path on disk + # + # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions def disk_path full_path end def ensure_storage_path_exist - gitlab_shell.add_namespace(repository_storage_path, namespace.full_path) + gitlab_shell.add_namespace(repository_storage_path, base_dir) end def rename_repo diff --git a/app/models/concerns/storage/uuid_project.rb b/app/models/concerns/storage/uuid_project.rb deleted file mode 100644 index 8a73287e518..00000000000 --- a/app/models/concerns/storage/uuid_project.rb +++ /dev/null @@ -1,58 +0,0 @@ -module Storage - module UUIDProject - extend ActiveSupport::Concern - - def uuid_dir - %Q(#{uuid[0..1]}/#{uuid[2..3]}) - end - - def disk_path - %Q(#{uuid_dir}/#{uuid}) - end - - def ensure_storage_path_exist - gitlab_shell.add_namespace(repository_storage_path, uuid_dir) - end - - def rename_repo - # TODO: We cannot wipe most of this method until we provide migration path for Container Registries - path_was = previous_changes['path'].first - old_path_with_namespace = File.join(namespace.full_path, path_was) - new_path_with_namespace = File.join(namespace.full_path, path) - - Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}" - - if has_container_registry_tags? - Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" - - # we currently doesn't support renaming repository if it contains images in container registry - raise StandardError.new('Project cannot be renamed, because images are present in its container registry') - end - - begin - # TODO: we can avoid cache expiration if cache is based on UUID or just project_id - expire_caches_before_rename(old_path_with_namespace) - expires_full_path_cache - - send_move_instructions(old_path_with_namespace) - - @old_path_with_namespace = old_path_with_namespace - - SystemHooksService.new.execute_hooks_for(self, :rename) - - @repository = nil - rescue => e - Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" - # Returning false does not rollback after_* transaction but gives - # us information about failing some of tasks - false - end - - Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" - - # TODO: When we move Uploads and Pages to use UUID we can disable this transfers as well - Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path) - Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path) - end - end -end diff --git a/app/models/project.rb b/app/models/project.rb index f0ea83dda0c..0e000df9790 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1424,10 +1424,12 @@ class Project < ActiveRecord::Base private def load_storage - if self.storage_version > 1 - self.class.include Storage::UUIDProject + return unless has_attribute?(:storage_version) + + if self.storage_version && self.storage_version >= 1 + self.extend Storage::HashedProject else - self.class.include Storage::LegacyProject + self.extend Storage::LegacyProject end end diff --git a/db/migrate/20170802013652_add_storage_fields_to_project.rb b/db/migrate/20170802013652_add_storage_fields_to_project.rb index 269103cd472..a0815da0fcd 100644 --- a/db/migrate/20170802013652_add_storage_fields_to_project.rb +++ b/db/migrate/20170802013652_add_storage_fields_to_project.rb @@ -4,36 +4,16 @@ class AddStorageFieldsToProject < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers - # Set this constant to true if this migration requires downtime. DOWNTIME = false - - # When a migration requires downtime you **must** uncomment the following - # constant and define a short and easy to understand explanation as to why the - # migration requires downtime. - # DOWNTIME_REASON = '' - - # When using the methods "add_concurrent_index", "remove_concurrent_index" or - # "add_column_with_default" you must disable the use of transactions - # as these methods can not run in an existing transaction. - # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure - # that either of them is the _only_ method called in the migration, - # any other changes should go in a separate migration. - # This ensures that upon failure _only_ the index creation or removing fails - # and can be retried or reverted easily. - # - # To disable transactions uncomment the following line and remove these - # comments: disable_ddl_transaction! def up # rubocop:disable Migration/AddColumnWithDefaultToLargeTable - add_column :projects, :uuid, :uuid - add_column_with_default :projects, :storage_version, :integer, default: 0, limit: 1 - add_concurrent_index :projects, :uuid + add_column :projects, :storage_version, :integer, limit: 2 + add_concurrent_index :projects, :storage_version end def down - remove_column :projects, :uuid remove_column :projects, :storage_version end end diff --git a/db/schema.rb b/db/schema.rb index dcd9532e4be..5a85a00bb12 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1208,8 +1208,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do t.datetime "last_repository_updated_at" t.string "ci_config_path" t.text "delete_error" - t.uuid "uuid" - t.integer "storage_version", limit: 2, default: 0, null: false + t.integer "storage_version", limit: 2 end add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree @@ -1226,7 +1225,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do add_index "projects", ["pending_delete"], name: "index_projects_on_pending_delete", using: :btree add_index "projects", ["runners_token"], name: "index_projects_on_runners_token", using: :btree add_index "projects", ["star_count"], name: "index_projects_on_star_count", using: :btree - add_index "projects", ["uuid"], name: "index_projects_on_uuid", using: :btree + add_index "projects", ["storage_version"], name: "index_projects_on_storage_version", using: :btree add_index "projects", ["visibility_level"], name: "index_projects_on_visibility_level", using: :btree create_table "protected_branch_merge_access_levels", force: :cascade do |t| diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 9d9ebcb389a..894950e341f 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -98,6 +98,7 @@ excluded_attributes: - :last_activity_at - :last_repository_updated_at - :last_repository_check_at + - :storage_version snippets: - :expired_at merge_request_diff: diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb index 4a2034b31b3..c6f1da82f3c 100644 --- a/spec/factories/projects.rb +++ b/spec/factories/projects.rb @@ -81,6 +81,10 @@ FactoryGirl.define do archived true end + trait :hashed do + storage_version 1 + end + trait :access_requestable do request_access_enabled true end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 5e60511f3a8..e6fde833c6b 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1251,60 +1251,6 @@ describe Project do end end - describe '#rename_repo' do - let(:project) { create(:project, :repository) } - let(:gitlab_shell) { Gitlab::Shell.new } - - before do - # Project#gitlab_shell returns a new instance of Gitlab::Shell on every - # call. This makes testing a bit easier. - allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) - allow(project).to receive(:previous_changes).and_return('path' => ['foo']) - end - - it 'renames a repository' do - stub_container_registry_config(enabled: false) - - expect(gitlab_shell).to receive(:mv_repository) - .ordered - .with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}") - .and_return(true) - - expect(gitlab_shell).to receive(:mv_repository) - .ordered - .with(project.repository_storage_path, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki") - .and_return(true) - - expect_any_instance_of(SystemHooksService) - .to receive(:execute_hooks_for) - .with(project, :rename) - - expect_any_instance_of(Gitlab::UploadsTransfer) - .to receive(:rename_project) - .with('foo', project.path, project.namespace.full_path) - - expect(project).to receive(:expire_caches_before_rename) - - expect(project).to receive(:expires_full_path_cache) - - project.rename_repo - end - - context 'container registry with images' do - let(:container_repository) { create(:container_repository) } - - before do - stub_container_registry_config(enabled: true) - stub_container_registry_tags(repository: :any, tags: ['tag']) - project.container_repositories << container_repository - end - - subject { project.rename_repo } - - it { expect {subject}.to raise_error(StandardError) } - end - end - describe '#expire_caches_before_rename' do let(:project) { create(:project, :repository) } let(:repo) { double(:repo, exists?: true) } @@ -2367,4 +2313,163 @@ describe Project do expect(project.forks_count).to eq(1) end end + + context 'legacy storage' do + let(:project) { create(:project, :repository) } + let(:gitlab_shell) { Gitlab::Shell.new } + + describe '#base_dir' do + it 'returns base_dir based on namespace only' do + expect(project.base_dir).to eq(project.namespace.full_path) + end + end + + describe '#disk_path' do + it 'returns disk_path based on namespace and project path' do + expect(project.disk_path).to eq("#{project.namespace.full_path}/#{project.path}") + end + end + + describe '#ensure_storage_path_exist' do + before do + allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) + end + + it 'delegates to gitlab_shell to ensure namespace is created' do + expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, project.base_dir) + + project.ensure_storage_path_exist + end + end + + describe '#rename_repo' do + before do + # Project#gitlab_shell returns a new instance of Gitlab::Shell on every + # call. This makes testing a bit easier. + allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) + allow(project).to receive(:previous_changes).and_return('path' => ['foo']) + end + + it 'renames a repository' do + stub_container_registry_config(enabled: false) + + expect(gitlab_shell).to receive(:mv_repository) + .ordered + .with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}") + .and_return(true) + + expect(gitlab_shell).to receive(:mv_repository) + .ordered + .with(project.repository_storage_path, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki") + .and_return(true) + + expect_any_instance_of(SystemHooksService) + .to receive(:execute_hooks_for) + .with(project, :rename) + + expect_any_instance_of(Gitlab::UploadsTransfer) + .to receive(:rename_project) + .with('foo', project.path, project.namespace.full_path) + + expect(project).to receive(:expire_caches_before_rename) + + expect(project).to receive(:expires_full_path_cache) + + project.rename_repo + end + + context 'container registry with images' do + let(:container_repository) { create(:container_repository) } + + before do + stub_container_registry_config(enabled: true) + stub_container_registry_tags(repository: :any, tags: ['tag']) + project.container_repositories << container_repository + end + + subject { project.rename_repo } + + it { expect{subject}.to raise_error(StandardError) } + end + end + end + + context 'hashed storage' do + let(:project) { create(:project, :repository, :hashed) } + let(:gitlab_shell) { Gitlab::Shell.new } + let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' } + + before do + allow(Digest::SHA2).to receive(:hexdigest) { hash } + end + + describe '#base_dir' do + it 'returns base_dir based on hash of project id' do + expect(project.base_dir).to eq('6b/86') + end + end + + describe '#disk_path' do + it 'returns disk_path based on has of project id' do + hashed_path = '6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' + + expect(project.disk_path).to eq(hashed_path) + end + end + + describe '#ensure_storage_path_exist' do + before do + allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) + end + + it 'delegates to gitlab_shell to ensure namespace is created' do + expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, '6b/86') + + project.ensure_storage_path_exist + end + end + + describe '#rename_repo' do + before do + # Project#gitlab_shell returns a new instance of Gitlab::Shell on every + # call. This makes testing a bit easier. + allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) + allow(project).to receive(:previous_changes).and_return('path' => ['foo']) + end + + it 'renames a repository' do + stub_container_registry_config(enabled: false) + + expect(gitlab_shell).not_to receive(:mv_repository) + + expect_any_instance_of(SystemHooksService) + .to receive(:execute_hooks_for) + .with(project, :rename) + + expect_any_instance_of(Gitlab::UploadsTransfer) + .to receive(:rename_project) + .with('foo', project.path, project.namespace.full_path) + + expect(project).to receive(:expire_caches_before_rename) + + expect(project).to receive(:expires_full_path_cache) + + project.rename_repo + end + + context 'container registry with images' do + let(:container_repository) { create(:container_repository) } + + before do + stub_container_registry_config(enabled: true) + stub_container_registry_tags(repository: :any, tags: ['tag']) + project.container_repositories << container_repository + end + + subject { project.rename_repo } + + it { expect{subject}.to raise_error(StandardError) } + end + end + end end -- cgit v1.2.1 From 72250a4ed8978b32c2e12dd05fc6feb8132e4083 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 7 Aug 2017 10:53:30 +0200 Subject: Enable automatic hashed storage for new projects by application settings --- app/helpers/application_settings_helper.rb | 1 + app/models/project.rb | 4 ++++ app/views/admin/application_settings/_form.html.haml | 11 +++++++++++ .../20170807071105_add_hashed_storage_to_settings.rb | 18 ++++++++++++++++++ db/schema.rb | 1 + 5 files changed, 35 insertions(+) create mode 100644 db/migrate/20170807071105_add_hashed_storage_to_settings.rb diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 150188f0b65..3b76da238e0 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -116,6 +116,7 @@ module ApplicationSettingsHelper :email_author_in_body, :enabled_git_access_protocol, :gravatar_enabled, + :hashed_storage_enabled, :help_page_hide_commercial_content, :help_page_support_url, :help_page_text, diff --git a/app/models/project.rb b/app/models/project.rb index 0e000df9790..3e26c4b2e0f 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -24,6 +24,7 @@ class Project < ActiveRecord::Base NUMBER_OF_PERMITTED_BOARDS = 1 UNKNOWN_IMPORT_URL = 'http://unknown.git'.freeze + LATEST_STORAGE_VERSION = 1 cache_markdown_field :description, pipeline: :description @@ -1428,6 +1429,9 @@ class Project < ActiveRecord::Base if self.storage_version && self.storage_version >= 1 self.extend Storage::HashedProject + elsif !self.persisted? && current_application_settings.hashed_storage_enabled + self.storage_version = LATEST_STORAGE_VERSION + self.extend Storage::HashedProject else self.extend Storage::LegacyProject end diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index dc585054316..124bbebbb78 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -492,6 +492,16 @@ %fieldset %legend Repository Storage + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :hashed_storage_enabled do + = f.check_box :hashed_storage_enabled + Create new projects using hashed storage paths + .help-block + Enable immutable, hash based paths and repository names to store repositories on disk. This prevents + repositories from having to be moved or renamed when the Project URL changes and may improve disk I/O performance. + %em (EXPERIMENTAL) .form-group = f.label :repository_storages, 'Storage paths for new projects', class: 'control-label col-sm-2' .col-sm-10 @@ -501,6 +511,7 @@ = succeed "." do = link_to "repository storages documentation", help_page_path("administration/repository_storages") + %fieldset %legend Repository Checks .form-group diff --git a/db/migrate/20170807071105_add_hashed_storage_to_settings.rb b/db/migrate/20170807071105_add_hashed_storage_to_settings.rb new file mode 100644 index 00000000000..0846557add8 --- /dev/null +++ b/db/migrate/20170807071105_add_hashed_storage_to_settings.rb @@ -0,0 +1,18 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddHashedStorageToSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default :application_settings, :hashed_storage_enabled, :boolean, default: false + end + + def down + remove_columns :application_settings, :hashed_storage_enabled + end +end diff --git a/db/schema.rb b/db/schema.rb index 5a85a00bb12..4d999ead607 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -128,6 +128,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do t.integer "performance_bar_allowed_group_id" t.boolean "password_authentication_enabled" t.boolean "project_export_enabled", default: true, null: false + t.boolean "hashed_storage_enabled", default: false, null: false end create_table "audit_events", force: :cascade do |t| -- cgit v1.2.1 From e7a060321fed61085c7d70d23e9ea33825d1467f Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 7 Aug 2017 11:07:42 +0200 Subject: Moving away from the "extend" based factory to a more traditional one. Using `extend` dynamically can lead to bad performance as it invalidates the method's cache. --- app/models/concerns/storage/hashed_project.rb | 72 ------------------- app/models/concerns/storage/legacy_project.rb | 73 -------------------- app/models/project.rb | 24 ++++--- app/models/storage/hashed_project.rb | 77 +++++++++++++++++++++ app/models/storage/legacy_project.rb | 80 ++++++++++++++++++++++ ...20170802013652_add_storage_fields_to_project.rb | 1 - spec/models/project_spec.rb | 3 +- 7 files changed, 174 insertions(+), 156 deletions(-) delete mode 100644 app/models/concerns/storage/hashed_project.rb delete mode 100644 app/models/concerns/storage/legacy_project.rb create mode 100644 app/models/storage/hashed_project.rb create mode 100644 app/models/storage/legacy_project.rb diff --git a/app/models/concerns/storage/hashed_project.rb b/app/models/concerns/storage/hashed_project.rb deleted file mode 100644 index 292a73903b5..00000000000 --- a/app/models/concerns/storage/hashed_project.rb +++ /dev/null @@ -1,72 +0,0 @@ -module Storage - module HashedProject - extend ActiveSupport::Concern - - # Base directory - # - # @return [String] directory where repository is stored - def base_dir - %Q(#{disk_hash[0..1]}/#{disk_hash[2..3]}) if disk_hash - end - - # Disk path is used to build repository and project's wiki path on disk - # - # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions - def disk_path - %Q(#{base_dir}/#{disk_hash}) - end - - def ensure_storage_path_exist - gitlab_shell.add_namespace(repository_storage_path, base_dir) - end - - def rename_repo - # TODO: We cannot wipe most of this method until we provide migration path for Container Registries - path_was = previous_changes['path'].first - old_path_with_namespace = File.join(namespace.full_path, path_was) - new_path_with_namespace = File.join(namespace.full_path, path) - - Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}" - - if has_container_registry_tags? - Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" - - # we currently doesn't support renaming repository if it contains images in container registry - raise StandardError.new('Project cannot be renamed, because images are present in its container registry') - end - - begin - # TODO: we can avoid cache expiration if cache is based on UUID or just project_id - expire_caches_before_rename(old_path_with_namespace) - expires_full_path_cache - - send_move_instructions(old_path_with_namespace) - - @old_path_with_namespace = old_path_with_namespace - - SystemHooksService.new.execute_hooks_for(self, :rename) - - @repository = nil - rescue => e - Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" - # Returning false does not rollback after_* transaction but gives - # us information about failing some of tasks - false - end - - Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" - - # TODO: When we move Uploads and Pages to use UUID we can disable this transfers as well - Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path) - Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path) - end - - private - - # Generates the hash for the project path and name on disk - # If you need to refer to the repository on disk, use the `#disk_path` - def disk_hash - @disk_hash ||= Digest::SHA2.hexdigest(self.id.to_s) if self.id - end - end -end diff --git a/app/models/concerns/storage/legacy_project.rb b/app/models/concerns/storage/legacy_project.rb deleted file mode 100644 index 839bbcc76ea..00000000000 --- a/app/models/concerns/storage/legacy_project.rb +++ /dev/null @@ -1,73 +0,0 @@ -module Storage - module LegacyProject - extend ActiveSupport::Concern - - # Base directory - # - # @return [String] directory where repository is stored - def base_dir - namespace.full_path - end - - # Disk path is used to build repository and project's wiki path on disk - # - # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions - def disk_path - full_path - end - - def ensure_storage_path_exist - gitlab_shell.add_namespace(repository_storage_path, base_dir) - end - - def rename_repo - path_was = previous_changes['path'].first - old_path_with_namespace = File.join(namespace.full_path, path_was) - new_path_with_namespace = File.join(namespace.full_path, path) - - Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}" - - if has_container_registry_tags? - Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" - - # we currently doesn't support renaming repository if it contains images in container registry - raise StandardError.new('Project cannot be renamed, because images are present in its container registry') - end - - expire_caches_before_rename(old_path_with_namespace) - - if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) - # If repository moved successfully we need to send update instructions to users. - # However we cannot allow rollback since we moved repository - # So we basically we mute exceptions in next actions - begin - gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") - send_move_instructions(old_path_with_namespace) - expires_full_path_cache - - @old_path_with_namespace = old_path_with_namespace - - SystemHooksService.new.execute_hooks_for(self, :rename) - - @repository = nil - rescue => e - Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" - # Returning false does not rollback after_* transaction but gives - # us information about failing some of tasks - false - end - else - Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" - - # if we cannot move namespace directory we should rollback - # db changes in order to prevent out of sync between db and fs - raise StandardError.new('repository cannot be renamed') - end - - Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" - - Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path) - Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path) - end - end -end diff --git a/app/models/project.rb b/app/models/project.rb index 3e26c4b2e0f..5f510412015 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -32,6 +32,8 @@ class Project < ActiveRecord::Base :merge_requests_enabled?, :issues_enabled?, to: :project_feature, allow_nil: true + delegate :base_dir, :disk_path, :ensure_storage_path_exist, :rename_repo, to: :storage + default_value_for :archived, false default_value_for :visibility_level, gitlab_config_features.visibility_level default_value_for :container_registry_enabled, gitlab_config_features.container_registry @@ -59,7 +61,7 @@ class Project < ActiveRecord::Base after_validation :check_pending_delete # Storage specific hooks - after_initialize :load_storage + after_initialize :use_hashed_storage after_create :ensure_storage_path_exist after_save :ensure_storage_path_exist, if: :namespace_id_changed? @@ -1424,16 +1426,20 @@ class Project < ActiveRecord::Base private - def load_storage - return unless has_attribute?(:storage_version) + def storage + @storage ||= + if !has_attribute?(:storage_version) # during migration + Storage::LegacyProject.new(self) + elsif self.storage_version && self.storage_version >= 1 + Storage::HashedProject.new(self) + else + Storage::LegacyProject.new(self) + end + end - if self.storage_version && self.storage_version >= 1 - self.extend Storage::HashedProject - elsif !self.persisted? && current_application_settings.hashed_storage_enabled + def use_hashed_storage + if !self.persisted? && current_application_settings.hashed_storage_enabled self.storage_version = LATEST_STORAGE_VERSION - self.extend Storage::HashedProject - else - self.extend Storage::LegacyProject end end diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb new file mode 100644 index 00000000000..364794a4eef --- /dev/null +++ b/app/models/storage/hashed_project.rb @@ -0,0 +1,77 @@ +module Storage + class HashedProject + attr_accessor :project + delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project + + def initialize(project) + @project = project + end + + # Base directory + # + # @return [String] directory where repository is stored + def base_dir + %Q(#{disk_hash[0..1]}/#{disk_hash[2..3]}) if disk_hash + end + + # Disk path is used to build repository and project's wiki path on disk + # + # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions + def disk_path + %Q(#{base_dir}/#{disk_hash}) + end + + def ensure_storage_path_exist + gitlab_shell.add_namespace(repository_storage_path, base_dir) + end + + def rename_repo + # TODO: We cannot wipe most of this method until we provide migration path for Container Registries + path_was = project.previous_changes['path'].first + old_path_with_namespace = File.join(namespace.full_path, path_was) + new_path_with_namespace = File.join(namespace.full_path, project.path) + + Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}" + + if project.has_container_registry_tags? + Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" + + # we currently doesn't support renaming repository if it contains images in container registry + raise StandardError.new('Project cannot be renamed, because images are present in its container registry') + end + + begin + # TODO: we can avoid cache expiration if cache is based on UUID or just project_id + project.expire_caches_before_rename(old_path_with_namespace) + project.expires_full_path_cache + + project.send_move_instructions(old_path_with_namespace) + + @old_path_with_namespace = old_path_with_namespace + + SystemHooksService.new.execute_hooks_for(project, :rename) + + @repository = nil + rescue => e + Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" + # Returning false does not rollback after_* transaction but gives + # us information about failing some of tasks + false + end + + Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" + + # TODO: When we move Uploads and Pages to use UUID we can disable this transfers as well + Gitlab::UploadsTransfer.new.rename_project(path_was, project.path, namespace.full_path) + Gitlab::PagesTransfer.new.rename_project(path_was, project.path, namespace.full_path) + end + + private + + # Generates the hash for the project path and name on disk + # If you need to refer to the repository on disk, use the `#disk_path` + def disk_hash + @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id + end + end +end diff --git a/app/models/storage/legacy_project.rb b/app/models/storage/legacy_project.rb new file mode 100644 index 00000000000..57c340075ba --- /dev/null +++ b/app/models/storage/legacy_project.rb @@ -0,0 +1,80 @@ +module Storage + class LegacyProject + attr_accessor :project + delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project + + def initialize(project) + @project = project + end + + # Base directory + # + # @return [String] directory where repository is stored + def base_dir + namespace.full_path + end + + # Disk path is used to build repository and project's wiki path on disk + # + # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions + def disk_path + project.full_path + end + + def ensure_storage_path_exist + return unless namespace + + gitlab_shell.add_namespace(repository_storage_path, base_dir) + end + + def rename_repo + path_was = project.previous_changes['path'].first + old_path_with_namespace = File.join(base_dir, path_was) + new_path_with_namespace = File.join(base_dir, project.path) + + Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}" + + if project.has_container_registry_tags? + Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" + + # we currently doesn't support renaming repository if it contains images in container registry + raise StandardError.new('Project cannot be renamed, because images are present in its container registry') + end + + project.expire_caches_before_rename(old_path_with_namespace) + + if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) + # If repository moved successfully we need to send update instructions to users. + # However we cannot allow rollback since we moved repository + # So we basically we mute exceptions in next actions + begin + gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") + project.send_move_instructions(old_path_with_namespace) + project.expires_full_path_cache + + @old_path_with_namespace = old_path_with_namespace + + SystemHooksService.new.execute_hooks_for(project, :rename) + + @repository = nil + rescue => e + Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" + # Returning false does not rollback after_* transaction but gives + # us information about failing some of tasks + false + end + else + Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" + + # if we cannot move namespace directory we should rollback + # db changes in order to prevent out of sync between db and fs + raise StandardError.new('repository cannot be renamed') + end + + Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" + + Gitlab::UploadsTransfer.new.rename_project(path_was, project.path, base_dir) + Gitlab::PagesTransfer.new.rename_project(path_was, project.path, base_dir) + end + end +end diff --git a/db/migrate/20170802013652_add_storage_fields_to_project.rb b/db/migrate/20170802013652_add_storage_fields_to_project.rb index a0815da0fcd..e99ae53ef11 100644 --- a/db/migrate/20170802013652_add_storage_fields_to_project.rb +++ b/db/migrate/20170802013652_add_storage_fields_to_project.rb @@ -8,7 +8,6 @@ class AddStorageFieldsToProject < ActiveRecord::Migration disable_ddl_transaction! def up - # rubocop:disable Migration/AddColumnWithDefaultToLargeTable add_column :projects, :storage_version, :integer, limit: 2 add_concurrent_index :projects, :storage_version end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index e6fde833c6b..8ff25a6cf6b 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2395,11 +2395,12 @@ describe Project do end context 'hashed storage' do - let(:project) { create(:project, :repository, :hashed) } + let(:project) { create(:project, :repository) } let(:gitlab_shell) { Gitlab::Shell.new } let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' } before do + stub_application_setting(hashed_storage_enabled: true) allow(Digest::SHA2).to receive(:hexdigest) { hash } end -- cgit v1.2.1 From 47d3ea01bbc197e6ec8c566f287961cef2ec1cb6 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 7 Aug 2017 11:44:01 +0200 Subject: Changelog --- changelogs/unreleased/28283-uuid-storage.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelogs/unreleased/28283-uuid-storage.yml diff --git a/changelogs/unreleased/28283-uuid-storage.yml b/changelogs/unreleased/28283-uuid-storage.yml new file mode 100644 index 00000000000..283e06d4b7f --- /dev/null +++ b/changelogs/unreleased/28283-uuid-storage.yml @@ -0,0 +1,4 @@ +--- +title: Hashed Storage support for Repositories (EXPERIMENTAL) +merge_request: 13246 +author: -- cgit v1.2.1 From 95a270c87104e1225d4c29a54611f5e4f7a76b56 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Tue, 8 Aug 2017 11:53:52 +0200 Subject: Fix repository reloading in some specs --- app/models/project.rb | 4 ++++ app/models/storage/hashed_project.rb | 4 ++-- app/models/storage/legacy_project.rb | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index 5f510412015..b84d06fef1e 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -478,6 +478,10 @@ class Project < ActiveRecord::Base @repository ||= Repository.new(full_path, self, disk_path: disk_path) end + def reload_repository! + @repository = nil + end + def container_registry_url if Gitlab.config.registry.enabled "#{Gitlab.config.registry.host_port}/#{full_path.downcase}" diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb index 364794a4eef..91f74b084b6 100644 --- a/app/models/storage/hashed_project.rb +++ b/app/models/storage/hashed_project.rb @@ -47,11 +47,11 @@ module Storage project.send_move_instructions(old_path_with_namespace) - @old_path_with_namespace = old_path_with_namespace + project.old_path_with_namespace = old_path_with_namespace SystemHooksService.new.execute_hooks_for(project, :rename) - @repository = nil + project.reload_repository! rescue => e Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" # Returning false does not rollback after_* transaction but gives diff --git a/app/models/storage/legacy_project.rb b/app/models/storage/legacy_project.rb index 57c340075ba..b7b073ad077 100644 --- a/app/models/storage/legacy_project.rb +++ b/app/models/storage/legacy_project.rb @@ -52,11 +52,11 @@ module Storage project.send_move_instructions(old_path_with_namespace) project.expires_full_path_cache - @old_path_with_namespace = old_path_with_namespace + project.old_path_with_namespace = old_path_with_namespace SystemHooksService.new.execute_hooks_for(project, :rename) - @repository = nil + project.reload_repository! rescue => e Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" # Returning false does not rollback after_* transaction but gives -- cgit v1.2.1 From fff5ebdcae6794de35f8eaff15217d8643c83686 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 9 Aug 2017 08:41:32 +0200 Subject: Removed some useless code, codestyle changes and removed an index --- app/models/project.rb | 6 ++---- app/models/storage/hashed_project.rb | 4 ++-- app/views/admin/application_settings/_form.html.haml | 2 +- db/migrate/20170802013652_add_storage_fields_to_project.rb | 2 -- db/schema.rb | 1 - spec/models/project_spec.rb | 4 ++-- 6 files changed, 7 insertions(+), 12 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index b84d06fef1e..d9e4d4d192e 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1432,9 +1432,7 @@ class Project < ActiveRecord::Base def storage @storage ||= - if !has_attribute?(:storage_version) # during migration - Storage::LegacyProject.new(self) - elsif self.storage_version && self.storage_version >= 1 + if self.storage_version && self.storage_version >= 1 Storage::HashedProject.new(self) else Storage::LegacyProject.new(self) @@ -1442,7 +1440,7 @@ class Project < ActiveRecord::Base end def use_hashed_storage - if !self.persisted? && current_application_settings.hashed_storage_enabled + if self.new_record? && current_application_settings.hashed_storage_enabled self.storage_version = LATEST_STORAGE_VERSION end end diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb index 91f74b084b6..e6d68a177fe 100644 --- a/app/models/storage/hashed_project.rb +++ b/app/models/storage/hashed_project.rb @@ -11,14 +11,14 @@ module Storage # # @return [String] directory where repository is stored def base_dir - %Q(#{disk_hash[0..1]}/#{disk_hash[2..3]}) if disk_hash + "#{disk_hash[0..1]}/#{disk_hash[2..3]}" if disk_hash end # Disk path is used to build repository and project's wiki path on disk # # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions def disk_path - %Q(#{base_dir}/#{disk_hash}) + "#{base_dir}/#{disk_hash}" end def ensure_storage_path_exist diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index 124bbebbb78..959af5c0d13 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -499,7 +499,7 @@ = f.check_box :hashed_storage_enabled Create new projects using hashed storage paths .help-block - Enable immutable, hash based paths and repository names to store repositories on disk. This prevents + Enable immutable, hash-based paths and repository names to store repositories on disk. This prevents repositories from having to be moved or renamed when the Project URL changes and may improve disk I/O performance. %em (EXPERIMENTAL) .form-group diff --git a/db/migrate/20170802013652_add_storage_fields_to_project.rb b/db/migrate/20170802013652_add_storage_fields_to_project.rb index e99ae53ef11..c2381a9d0b2 100644 --- a/db/migrate/20170802013652_add_storage_fields_to_project.rb +++ b/db/migrate/20170802013652_add_storage_fields_to_project.rb @@ -5,11 +5,9 @@ class AddStorageFieldsToProject < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers DOWNTIME = false - disable_ddl_transaction! def up add_column :projects, :storage_version, :integer, limit: 2 - add_concurrent_index :projects, :storage_version end def down diff --git a/db/schema.rb b/db/schema.rb index 4d999ead607..cd488630237 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1226,7 +1226,6 @@ ActiveRecord::Schema.define(version: 20170820100558) do add_index "projects", ["pending_delete"], name: "index_projects_on_pending_delete", using: :btree add_index "projects", ["runners_token"], name: "index_projects_on_runners_token", using: :btree add_index "projects", ["star_count"], name: "index_projects_on_star_count", using: :btree - add_index "projects", ["storage_version"], name: "index_projects_on_storage_version", using: :btree add_index "projects", ["visibility_level"], name: "index_projects_on_visibility_level", using: :btree create_table "protected_branch_merge_access_levels", force: :cascade do |t| diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 8ff25a6cf6b..ec620070cdf 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2389,7 +2389,7 @@ describe Project do subject { project.rename_repo } - it { expect{subject}.to raise_error(StandardError) } + it { expect { subject }.to raise_error(StandardError) } end end end @@ -2469,7 +2469,7 @@ describe Project do subject { project.rename_repo } - it { expect{subject}.to raise_error(StandardError) } + it { expect { subject }.to raise_error(StandardError) } end end end -- cgit v1.2.1 From de89dcc2137039bbdb1811e03715e8e62adc8c66 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Fri, 11 Aug 2017 09:51:56 +0200 Subject: Some codestyle changes and fixes for GitLab pages --- app/models/project.rb | 3 ++- spec/models/project_spec.rb | 27 ++++++++++++++++++--------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index d9e4d4d192e..833ced08e81 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1222,7 +1222,8 @@ class Project < ActiveRecord::Base end def pages_path - File.join(Settings.pages.path, disk_path) + # TODO: when we migrate Pages to work with new storage types, change here to use disk_path + File.join(Settings.pages.path, full_path) end def public_pages_path diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index ec620070cdf..87f31e2a588 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2318,6 +2318,10 @@ describe Project do let(:project) { create(:project, :repository) } let(:gitlab_shell) { Gitlab::Shell.new } + before do + allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) + end + describe '#base_dir' do it 'returns base_dir based on namespace only' do expect(project.base_dir).to eq(project.namespace.full_path) @@ -2331,10 +2335,6 @@ describe Project do end describe '#ensure_storage_path_exist' do - before do - allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) - end - it 'delegates to gitlab_shell to ensure namespace is created' do expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, project.base_dir) @@ -2392,6 +2392,12 @@ describe Project do it { expect { subject }.to raise_error(StandardError) } end end + + describe '#pages_path' do + it 'returns a path where pages are stored' do + expect(project.pages_path).to eq(File.join(Settings.pages.path, project.namespace.full_path, project.path)) + end + end end context 'hashed storage' do @@ -2402,6 +2408,7 @@ describe Project do before do stub_application_setting(hashed_storage_enabled: true) allow(Digest::SHA2).to receive(:hexdigest) { hash } + allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) end describe '#base_dir' do @@ -2411,7 +2418,7 @@ describe Project do end describe '#disk_path' do - it 'returns disk_path based on has of project id' do + it 'returns disk_path based on hash of project id' do hashed_path = '6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' expect(project.disk_path).to eq(hashed_path) @@ -2419,10 +2426,6 @@ describe Project do end describe '#ensure_storage_path_exist' do - before do - allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) - end - it 'delegates to gitlab_shell to ensure namespace is created' do expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, '6b/86') @@ -2472,5 +2475,11 @@ describe Project do it { expect { subject }.to raise_error(StandardError) } end end + + describe '#pages_path' do + it 'returns a path where pages are stored' do + expect(project.pages_path).to eq(File.join(Settings.pages.path, project.namespace.full_path, project.path)) + end + end end end -- cgit v1.2.1 From 78823675b24e82e73a523ad98f1dec78bec6976c Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Fri, 11 Aug 2017 12:03:35 +0200 Subject: Prevent using gitlab import task when hashed storage is enabled --- lib/tasks/gitlab/import.rake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 48bd9139ce8..6e10ba374bf 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -11,6 +11,12 @@ namespace :gitlab do # desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do + if Project.current_application_settings.hashed_storage_enabled + puts 'Cannot import repositories when Hashed Storage is enabled'.color(:red) + + exit 1 + end + Gitlab.config.repositories.storages.each_value do |repository_storage| git_base_path = repository_storage['path'] repos_to_import = Dir.glob(git_base_path + '/**/*.git') -- cgit v1.2.1 From d17a7be8308b06b7077a7cffc5d258148ee08c87 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 16 Aug 2017 04:49:54 +0200 Subject: Refactor project and storage types --- app/models/project.rb | 54 ++++++++++++++++++++-- app/models/storage/hashed_project.rb | 43 ++--------------- app/models/storage/legacy_project.rb | 45 ++++-------------- lib/backup/repository.rb | 2 +- ...p_namespaceless_pending_delete_projects_spec.rb | 2 +- spec/models/project_spec.rb | 8 ++-- .../namespaceless_project_destroy_worker_spec.rb | 2 +- 7 files changed, 69 insertions(+), 87 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index 833ced08e81..a13ad9dceec 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -32,7 +32,7 @@ class Project < ActiveRecord::Base :merge_requests_enabled?, :issues_enabled?, to: :project_feature, allow_nil: true - delegate :base_dir, :disk_path, :ensure_storage_path_exist, :rename_repo, to: :storage + delegate :base_dir, :disk_path, :ensure_storage_path_exists, to: :storage default_value_for :archived, false default_value_for :visibility_level, gitlab_config_features.visibility_level @@ -62,8 +62,8 @@ class Project < ActiveRecord::Base # Storage specific hooks after_initialize :use_hashed_storage - after_create :ensure_storage_path_exist - after_save :ensure_storage_path_exist, if: :namespace_id_changed? + after_create :ensure_storage_path_exists + after_save :ensure_storage_path_exists, if: :namespace_id_changed? acts_as_taggable @@ -1262,6 +1262,50 @@ class Project < ActiveRecord::Base end end + def rename_repo + new_full_path = build_full_path + + Rails.logger.error "Attempting to rename #{full_path_was} -> #{new_full_path}" + + if has_container_registry_tags? + Rails.logger.error "Project #{full_path_was} cannot be renamed because container registry tags are present!" + + # we currently doesn't support renaming repository if it contains images in container registry + raise StandardError.new('Project cannot be renamed, because images are present in its container registry') + end + + expire_caches_before_rename(full_path_was) + + if storage.rename_repo + Gitlab::AppLogger.info "Project was renamed: #{full_path_was} -> #{new_full_path}" + rename_repo_notify! + after_rename_repo + else + Rails.logger.error "Repository could not be renamed: #{full_path_was} -> #{new_full_path}" + + # if we cannot move namespace directory we should rollback + # db changes in order to prevent out of sync between db and fs + raise StandardError.new('repository cannot be renamed') + end + end + + def rename_repo_notify! + send_move_instructions(full_path_was) + expires_full_path_cache + + self.old_path_with_namespace = full_path_was + SystemHooksService.new.execute_hooks_for(self, :rename) + + reload_repository! + end + + def after_rename_repo + path_before_change = previous_changes['path'].first + + Gitlab::UploadsTransfer.new.rename_project(path_before_change, self.path, namespace.full_path) + Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path) + end + def running_or_pending_build_count(force: false) Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do builds.running_or_pending.count(:all) @@ -1420,6 +1464,10 @@ class Project < ActiveRecord::Base end end + def full_path_was + File.join(namespace.full_path, previous_changes['path'].first) + end + alias_method :name_with_namespace, :full_name alias_method :human_name, :full_name # @deprecated cannot remove yet because it has an index with its name in elasticsearch diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb index e6d68a177fe..1a10e0e59a8 100644 --- a/app/models/storage/hashed_project.rb +++ b/app/models/storage/hashed_project.rb @@ -18,52 +18,15 @@ module Storage # # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions def disk_path - "#{base_dir}/#{disk_hash}" + "#{base_dir}/#{disk_hash}" if disk_hash end - def ensure_storage_path_exist + def ensure_storage_path_exists gitlab_shell.add_namespace(repository_storage_path, base_dir) end def rename_repo - # TODO: We cannot wipe most of this method until we provide migration path for Container Registries - path_was = project.previous_changes['path'].first - old_path_with_namespace = File.join(namespace.full_path, path_was) - new_path_with_namespace = File.join(namespace.full_path, project.path) - - Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}" - - if project.has_container_registry_tags? - Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" - - # we currently doesn't support renaming repository if it contains images in container registry - raise StandardError.new('Project cannot be renamed, because images are present in its container registry') - end - - begin - # TODO: we can avoid cache expiration if cache is based on UUID or just project_id - project.expire_caches_before_rename(old_path_with_namespace) - project.expires_full_path_cache - - project.send_move_instructions(old_path_with_namespace) - - project.old_path_with_namespace = old_path_with_namespace - - SystemHooksService.new.execute_hooks_for(project, :rename) - - project.reload_repository! - rescue => e - Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" - # Returning false does not rollback after_* transaction but gives - # us information about failing some of tasks - false - end - - Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" - - # TODO: When we move Uploads and Pages to use UUID we can disable this transfers as well - Gitlab::UploadsTransfer.new.rename_project(path_was, project.path, namespace.full_path) - Gitlab::PagesTransfer.new.rename_project(path_was, project.path, namespace.full_path) + true end private diff --git a/app/models/storage/legacy_project.rb b/app/models/storage/legacy_project.rb index b7b073ad077..9d9e5e1d352 100644 --- a/app/models/storage/legacy_project.rb +++ b/app/models/storage/legacy_project.rb @@ -21,60 +21,31 @@ module Storage project.full_path end - def ensure_storage_path_exist + def ensure_storage_path_exists return unless namespace gitlab_shell.add_namespace(repository_storage_path, base_dir) end def rename_repo - path_was = project.previous_changes['path'].first - old_path_with_namespace = File.join(base_dir, path_was) - new_path_with_namespace = File.join(base_dir, project.path) + new_full_path = project.build_full_path - Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}" - - if project.has_container_registry_tags? - Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" - - # we currently doesn't support renaming repository if it contains images in container registry - raise StandardError.new('Project cannot be renamed, because images are present in its container registry') - end - - project.expire_caches_before_rename(old_path_with_namespace) - - if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) + if gitlab_shell.mv_repository(repository_storage_path, project.full_path_was, new_full_path) # If repository moved successfully we need to send update instructions to users. # However we cannot allow rollback since we moved repository # So we basically we mute exceptions in next actions begin - gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") - project.send_move_instructions(old_path_with_namespace) - project.expires_full_path_cache - - project.old_path_with_namespace = old_path_with_namespace - - SystemHooksService.new.execute_hooks_for(project, :rename) - - project.reload_repository! + gitlab_shell.mv_repository(repository_storage_path, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki") + return true rescue => e - Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" + Rails.logger.error "Exception renaming #{project.full_path_was} -> #{new_full_path}: #{e}" # Returning false does not rollback after_* transaction but gives # us information about failing some of tasks - false + return false end - else - Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" - - # if we cannot move namespace directory we should rollback - # db changes in order to prevent out of sync between db and fs - raise StandardError.new('repository cannot be renamed') end - Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" - - Gitlab::UploadsTransfer.new.rename_project(path_was, project.path, base_dir) - Gitlab::PagesTransfer.new.rename_project(path_was, project.path, base_dir) + false end end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 88821ae56e0..4e92be85110 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -75,7 +75,7 @@ module Backup path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) - project.ensure_storage_path_exist + project.ensure_storage_path_exists cmd = if File.exist?(path_to_project_bundle) %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) diff --git a/spec/migrations/cleanup_namespaceless_pending_delete_projects_spec.rb b/spec/migrations/cleanup_namespaceless_pending_delete_projects_spec.rb index 12cac1d033d..b47f3314926 100644 --- a/spec/migrations/cleanup_namespaceless_pending_delete_projects_spec.rb +++ b/spec/migrations/cleanup_namespaceless_pending_delete_projects_spec.rb @@ -4,7 +4,7 @@ require Rails.root.join('db', 'post_migrate', '20170502101023_cleanup_namespacel describe CleanupNamespacelessPendingDeleteProjects do before do # Stub after_save callbacks that will fail when Project has no namespace - allow_any_instance_of(Project).to receive(:ensure_storage_path_exist).and_return(nil) + allow_any_instance_of(Project).to receive(:ensure_storage_path_exists).and_return(nil) allow_any_instance_of(Project).to receive(:update_project_statistics).and_return(nil) end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 87f31e2a588..6b646393696 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2334,11 +2334,11 @@ describe Project do end end - describe '#ensure_storage_path_exist' do + describe '#ensure_storage_path_exists' do it 'delegates to gitlab_shell to ensure namespace is created' do expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, project.base_dir) - project.ensure_storage_path_exist + project.ensure_storage_path_exists end end @@ -2425,11 +2425,11 @@ describe Project do end end - describe '#ensure_storage_path_exist' do + describe '#ensure_storage_path_exists' do it 'delegates to gitlab_shell to ensure namespace is created' do expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, '6b/86') - project.ensure_storage_path_exist + project.ensure_storage_path_exists end end diff --git a/spec/workers/namespaceless_project_destroy_worker_spec.rb b/spec/workers/namespaceless_project_destroy_worker_spec.rb index f2706254284..817e103fd9a 100644 --- a/spec/workers/namespaceless_project_destroy_worker_spec.rb +++ b/spec/workers/namespaceless_project_destroy_worker_spec.rb @@ -5,7 +5,7 @@ describe NamespacelessProjectDestroyWorker do before do # Stub after_save callbacks that will fail when Project has no namespace - allow_any_instance_of(Project).to receive(:ensure_storage_path_exist).and_return(nil) + allow_any_instance_of(Project).to receive(:ensure_storage_path_exists).and_return(nil) allow_any_instance_of(Project).to receive(:update_project_statistics).and_return(nil) end -- cgit v1.2.1 From 02737b8508eb3cce6d8c5ece1f2ffb4772c609a5 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Thu, 17 Aug 2017 17:39:18 +0200 Subject: Use `@hashed` prefix for hashed paths on disk, to avoid collision with existing ones --- app/models/storage/hashed_project.rb | 6 ++++-- spec/models/project_spec.rb | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb index 1a10e0e59a8..fae1b64961a 100644 --- a/app/models/storage/hashed_project.rb +++ b/app/models/storage/hashed_project.rb @@ -1,7 +1,9 @@ module Storage class HashedProject attr_accessor :project - delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project + delegate :gitlab_shell, :repository_storage_path, to: :project + + ROOT_PATH_PREFIX = '@hashed'.freeze def initialize(project) @project = project @@ -11,7 +13,7 @@ module Storage # # @return [String] directory where repository is stored def base_dir - "#{disk_hash[0..1]}/#{disk_hash[2..3]}" if disk_hash + "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}" if disk_hash end # Disk path is used to build repository and project's wiki path on disk diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 6b646393696..c7d2f2063af 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2413,13 +2413,13 @@ describe Project do describe '#base_dir' do it 'returns base_dir based on hash of project id' do - expect(project.base_dir).to eq('6b/86') + expect(project.base_dir).to eq('@hashed/6b/86') end end describe '#disk_path' do it 'returns disk_path based on hash of project id' do - hashed_path = '6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' + hashed_path = '@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' expect(project.disk_path).to eq(hashed_path) end @@ -2427,7 +2427,7 @@ describe Project do describe '#ensure_storage_path_exists' do it 'delegates to gitlab_shell to ensure namespace is created' do - expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, '6b/86') + expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, '@hashed/6b/86') project.ensure_storage_path_exists end -- cgit v1.2.1 From fb9e059a412d95ee3b7442a472ecfd9c67eafd7b Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 21 Aug 2017 15:28:07 +0200 Subject: Make sure repository's removal work for legacy and hashed storages --- app/models/project.rb | 4 ++++ app/services/groups/destroy_service.rb | 2 +- app/services/users/destroy_service.rb | 6 ++++-- spec/factories/projects.rb | 2 +- spec/models/project_spec.rb | 8 ++++++++ spec/services/groups/destroy_service_spec.rb | 26 ++++++++++++++++++++++++-- spec/services/users/destroy_service_spec.rb | 27 +++++++++++++++++++++++++-- 7 files changed, 67 insertions(+), 8 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index a13ad9dceec..37f4dd08355 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1477,6 +1477,10 @@ class Project < ActiveRecord::Base Projects::ForksCountService.new(self).count end + def legacy_storage? + self.storage_version.nil? + end + private def storage diff --git a/app/services/groups/destroy_service.rb b/app/services/groups/destroy_service.rb index f565612a89d..e3f9d9ee95d 100644 --- a/app/services/groups/destroy_service.rb +++ b/app/services/groups/destroy_service.rb @@ -13,7 +13,7 @@ module Groups # Execute the destruction of the models immediately to ensure atomic cleanup. # Skip repository removal because we remove directory with namespace # that contain all these repositories - ::Projects::DestroyService.new(project, current_user, skip_repo: true).execute + ::Projects::DestroyService.new(project, current_user, skip_repo: project.legacy_storage?).execute end group.children.each do |group| diff --git a/app/services/users/destroy_service.rb b/app/services/users/destroy_service.rb index 9d7237c2fbb..8e20de8dfa5 100644 --- a/app/services/users/destroy_service.rb +++ b/app/services/users/destroy_service.rb @@ -35,16 +35,18 @@ module Users Groups::DestroyService.new(group, current_user).execute end + namespace = user.namespace + namespace.prepare_for_destroy + user.personal_projects.each do |project| # Skip repository removal because we remove directory with namespace # that contain all this repositories - ::Projects::DestroyService.new(project, current_user, skip_repo: true).execute + ::Projects::DestroyService.new(project, current_user, skip_repo: project.legacy_storage?).execute end MigrateToGhostUserService.new(user).execute unless options[:hard_delete] # Destroy the namespace after destroying the user since certain methods may depend on the namespace existing - namespace = user.namespace user_data = user.destroy namespace.really_destroy! diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb index c6f1da82f3c..9ebda0ba03b 100644 --- a/spec/factories/projects.rb +++ b/spec/factories/projects.rb @@ -82,7 +82,7 @@ FactoryGirl.define do end trait :hashed do - storage_version 1 + storage_version Project::LATEST_STORAGE_VERSION end trait :access_requestable do diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index c7d2f2063af..2e613c44357 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2342,6 +2342,14 @@ describe Project do end end + describe '#legacy_storage?' do + it 'returns true when storage_version is nil' do + project = build(:project) + + expect(project.legacy_storage?).to be_truthy + end + end + describe '#rename_repo' do before do # Project#gitlab_shell returns a new instance of Gitlab::Shell on every diff --git a/spec/services/groups/destroy_service_spec.rb b/spec/services/groups/destroy_service_spec.rb index 1b2ce3cd03e..ac4b9c02ba7 100644 --- a/spec/services/groups/destroy_service_spec.rb +++ b/spec/services/groups/destroy_service_spec.rb @@ -8,8 +8,8 @@ describe Groups::DestroyService do let!(:nested_group) { create(:group, parent: group) } let!(:project) { create(:project, namespace: group) } let!(:notification_setting) { create(:notification_setting, source: group)} - let!(:gitlab_shell) { Gitlab::Shell.new } - let!(:remove_path) { group.path + "+#{group.id}+deleted" } + let(:gitlab_shell) { Gitlab::Shell.new } + let(:remove_path) { group.path + "+#{group.id}+deleted" } before do group.add_user(user, Gitlab::Access::OWNER) @@ -134,4 +134,26 @@ describe Groups::DestroyService do it_behaves_like 'group destruction', false end + + describe 'repository removal' do + before do + destroy_group(group, user, false) + end + + context 'legacy storage' do + let!(:project) { create(:project, :empty_repo, namespace: group) } + + it 'removes repository' do + expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey + end + end + + context 'hashed storage' do + let!(:project) { create(:project, :hashed, :empty_repo, namespace: group) } + + it 'removes repository' do + expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey + end + end + end end diff --git a/spec/services/users/destroy_service_spec.rb b/spec/services/users/destroy_service_spec.rb index a82567f6f43..58a5bede3de 100644 --- a/spec/services/users/destroy_service_spec.rb +++ b/spec/services/users/destroy_service_spec.rb @@ -4,9 +4,10 @@ describe Users::DestroyService do describe "Deletes a user and all their personal projects" do let!(:user) { create(:user) } let!(:admin) { create(:admin) } - let!(:namespace) { create(:namespace, owner: user) } + let!(:namespace) { user.namespace } let!(:project) { create(:project, namespace: namespace) } let(:service) { described_class.new(admin) } + let(:gitlab_shell) { Gitlab::Shell.new } context 'no options are given' do it 'deletes the user' do @@ -14,7 +15,7 @@ describe Users::DestroyService do expect { user_data['email'].to eq(user.email) } expect { User.find(user.id) }.to raise_error(ActiveRecord::RecordNotFound) - expect { Namespace.with_deleted.find(user.namespace.id) }.to raise_error(ActiveRecord::RecordNotFound) + expect { Namespace.with_deleted.find(namespace.id) }.to raise_error(ActiveRecord::RecordNotFound) end it 'will delete the project' do @@ -165,5 +166,27 @@ describe Users::DestroyService do expect(Issue.exists?(issue.id)).to be_falsy end end + + describe "user personal's repository removal" do + before do + Sidekiq::Testing.inline! { service.execute(user) } + end + + context 'legacy storage' do + let!(:project) { create(:project, :empty_repo, namespace: user.namespace) } + + it 'removes repository' do + expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey + end + end + + context 'hashed storage' do + let!(:project) { create(:project, :empty_repo, :hashed, namespace: user.namespace) } + + it 'removes repository' do + expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey + end + end + end end end -- cgit v1.2.1 From 6e8d0b78ebbde2eada151649fc7d1040b902e28f Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Tue, 15 Aug 2017 13:56:04 +0100 Subject: Use event-based waiting in Gitlab::JobWaiter --- app/workers/authorized_projects_worker.rb | 13 ++++-- lib/gitlab/job_waiter.rb | 57 +++++++++++++++++++++---- spec/lib/gitlab/job_waiter_spec.rb | 41 +++++++++++------- spec/workers/authorized_projects_worker_spec.rb | 16 ++++--- 4 files changed, 94 insertions(+), 33 deletions(-) diff --git a/app/workers/authorized_projects_worker.rb b/app/workers/authorized_projects_worker.rb index 13207a8bc71..be4c77503bb 100644 --- a/app/workers/authorized_projects_worker.rb +++ b/app/workers/authorized_projects_worker.rb @@ -4,18 +4,25 @@ class AuthorizedProjectsWorker # Schedules multiple jobs and waits for them to be completed. def self.bulk_perform_and_wait(args_list) - job_ids = bulk_perform_async(args_list) + waiter = Gitlab::JobWaiter.new(args_list.size) - Gitlab::JobWaiter.new(job_ids).wait + # Point all the bulk jobs at the same JobWaiter. Converts, [[1], [2], [3]] + # into [[1, "key"], [2, "key"], [3, "key"]] + waiting_args_list = args_list.map { |args| args << waiter.key } + bulk_perform_async(waiting_args_list) + + waiter.wait end def self.bulk_perform_async(args_list) Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], 'args' => args_list) end - def perform(user_id) + def perform(user_id, notify_key = nil) user = User.find_by(id: user_id) user&.refresh_authorized_projects + ensure + Gitlab::JobWaiter.notify(notify_key, jid) if notify_key end end diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 208f0e1bbea..4d6bbda15f3 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -1,12 +1,31 @@ module Gitlab # JobWaiter can be used to wait for a number of Sidekiq jobs to complete. + # + # Its use requires the cooperation of the sidekiq jobs themselves. Set up the + # waiter, then start the jobs, passing them its `key`. Their `perform` methods + # should look like: + # + # def perform(args, notify_key) + # # do work + # ensure + # ::Gitlab::JobWaiter.notify(notify_key, jid) + # end + # + # The JobWaiter blocks popping items from a Redis array. All the sidekiq jobs + # push to that array when done. Once the waiter has popped `count` items, it + # knows all the jobs are done. class JobWaiter - # The sleep interval between checking keys, in seconds. - INTERVAL = 0.1 + def self.notify(key, jid) + Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } + end + + attr_reader :key, :jobs_remaining, :finished - # jobs - The job IDs to wait for. - def initialize(jobs) - @jobs = jobs + # jobs_remaining - the number of jobs left to wait for + def initialize(jobs_remaining) + @key = "gitlab:job_waiter:#{SecureRandom.uuid}" + @jobs_remaining = jobs_remaining + @finished = [] end # Waits for all the jobs to be completed. @@ -15,13 +34,33 @@ module Gitlab # ensures we don't indefinitely block a caller in case a job takes # long to process, or is never processed. def wait(timeout = 10) - start = Time.current + deadline = Time.now.utc + timeout + + Gitlab::Redis::SharedState.with do |redis| + # Fallback key expiry: allow a long grace period to reduce the chance of + # a job pushing to an expired key and recreating it + redis.expire(key, [timeout * 2, 10.minutes.to_i].max) + + while jobs_remaining > 0 + # Redis will not take fractional seconds. Prefer waiting too long over + # not waiting long enough + seconds_left = (deadline - Time.now.utc).ceil - while (Time.current - start) <= timeout - break if SidekiqStatus.all_completed?(@jobs) + # Redis interprets 0 as "wait forever", so skip the final `blpop` call + break if seconds_left <= 0 - sleep(INTERVAL) # to not overload Redis too much. + list, jid = redis.blpop(key, timeout: seconds_left) + break unless list && jid # timed out + + @finished << jid + @jobs_remaining -= 1 + end + + # All jobs have finished, so expire the key immediately + redis.expire(key, 0) if jobs_remaining == 0 end + + finished end end end diff --git a/spec/lib/gitlab/job_waiter_spec.rb b/spec/lib/gitlab/job_waiter_spec.rb index 6186cec2689..b0b4fdc09bc 100644 --- a/spec/lib/gitlab/job_waiter_spec.rb +++ b/spec/lib/gitlab/job_waiter_spec.rb @@ -1,30 +1,39 @@ require 'spec_helper' describe Gitlab::JobWaiter do - describe '#wait' do - let(:waiter) { described_class.new(%w(a)) } - it 'returns when all jobs have been completed' do - expect(Gitlab::SidekiqStatus).to receive(:all_completed?).with(%w(a)) - .and_return(true) + describe '.notify' do + it 'pushes the jid to the named queue' do + key = 'gitlab:job_waiter:foo' + jid = 1 - expect(waiter).not_to receive(:sleep) + redis = double('redis') + expect(Gitlab::Redis::SharedState).to receive(:with).and_yield(redis) + expect(redis).to receive(:lpush).with(key, jid) - waiter.wait + described_class.notify(key, jid) end + end + + describe '#wait' do + let(:waiter) { described_class.new(2) } - it 'sleeps between checking the job statuses' do - expect(Gitlab::SidekiqStatus).to receive(:all_completed?) - .with(%w(a)) - .and_return(false, true) + it 'returns when all jobs have been completed' do + described_class.notify(waiter.key, 'a') + described_class.notify(waiter.key, 'b') - expect(waiter).to receive(:sleep).with(described_class::INTERVAL) + result = nil + expect { Timeout.timeout(1) { result = waiter.wait(2) } }.not_to raise_error - waiter.wait + expect(result).to contain_exactly('a', 'b') end - it 'returns when timing out' do - expect(waiter).not_to receive(:sleep) - waiter.wait(0) + it 'times out if not all jobs complete' do + described_class.notify(waiter.key, 'a') + + result = nil + expect { Timeout.timeout(2) { result = waiter.wait(1) } }.not_to raise_error + + expect(result).to contain_exactly('a') end end end diff --git a/spec/workers/authorized_projects_worker_spec.rb b/spec/workers/authorized_projects_worker_spec.rb index 03b9b99e263..f8385ae7c72 100644 --- a/spec/workers/authorized_projects_worker_spec.rb +++ b/spec/workers/authorized_projects_worker_spec.rb @@ -29,21 +29,27 @@ describe AuthorizedProjectsWorker do end describe '#perform' do - subject { described_class.new } + let(:user) { create(:user) } - it "refreshes user's authorized projects" do - user = create(:user) + subject(:job) { described_class.new } + it "refreshes user's authorized projects" do expect_any_instance_of(User).to receive(:refresh_authorized_projects) - subject.perform(user.id) + job.perform(user.id) + end + + it 'notifies the JobWaiter when done if the key is provided' do + expect(Gitlab::JobWaiter).to receive(:notify).with('notify-key', job.jid) + + job.perform(user.id, 'notify-key') end context "when the user is not found" do it "does nothing" do expect_any_instance_of(User).not_to receive(:refresh_authorized_projects) - subject.perform(-1) + job.perform(-1) end end end -- cgit v1.2.1