From bcd664f53a4009bc752fbc47e1c4d6f76c0b8cc2 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sun, 3 Jun 2018 15:04:47 +0900 Subject: Fix specs. Rename migration file name which was conflicted with background migration's. --- .../20180529152628_archive_legacy_traces.rb | 33 ---------------- ...0529152628_schedule_to_archive_legacy_traces.rb | 33 ++++++++++++++++ .../archive_legacy_traces_spec.rb | 35 ++++------------- spec/migrations/archive_legacy_traces_spec.rb | 45 ---------------------- .../schedule_to_archive_legacy_traces_spec.rb | 45 ++++++++++++++++++++++ spec/support/trace/trace_helpers.rb | 23 +++++++++++ 6 files changed, 109 insertions(+), 105 deletions(-) delete mode 100644 db/post_migrate/20180529152628_archive_legacy_traces.rb create mode 100644 db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb delete mode 100644 spec/migrations/archive_legacy_traces_spec.rb create mode 100644 spec/migrations/schedule_to_archive_legacy_traces_spec.rb create mode 100644 spec/support/trace/trace_helpers.rb diff --git a/db/post_migrate/20180529152628_archive_legacy_traces.rb b/db/post_migrate/20180529152628_archive_legacy_traces.rb deleted file mode 100644 index 12f219f606c..00000000000 --- a/db/post_migrate/20180529152628_archive_legacy_traces.rb +++ /dev/null @@ -1,33 +0,0 @@ -class ArchiveLegacyTraces < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - BATCH_SIZE = 5000 - BACKGROUND_MIGRATION_CLASS = 'ArchiveLegacyTraces' - - disable_ddl_transaction! - - class Build < ActiveRecord::Base - include EachBatch - self.table_name = 'ci_builds' - self.inheritance_column = :_type_disabled # Disable STI - - scope :finished, -> { where(status: [:success, :failed, :canceled]) } - - scope :without_archived_trace, -> do - where('NOT EXISTS (SELECT 1 FROM ci_job_artifacts WHERE ci_builds.id = ci_job_artifacts.job_id AND ci_job_artifacts.file_type = 3)') - end - end - - def up - queue_background_migration_jobs_by_range_at_intervals( - ::ArchiveLegacyTraces::Build.finished.without_archived_trace, - BACKGROUND_MIGRATION_CLASS, - 5.minutes, - batch_size: BATCH_SIZE) - end - - def down - # noop - end -end diff --git a/db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb b/db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb new file mode 100644 index 00000000000..ea782e1596b --- /dev/null +++ b/db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb @@ -0,0 +1,33 @@ +class ScheduleToArchiveLegacyTraces < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 5000 + BACKGROUND_MIGRATION_CLASS = 'ArchiveLegacyTraces' + + disable_ddl_transaction! + + class Build < ActiveRecord::Base + include EachBatch + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled # Disable STI + + scope :finished, -> { where(status: [:success, :failed, :canceled]) } + + scope :without_archived_trace, -> do + where('NOT EXISTS (SELECT 1 FROM ci_job_artifacts WHERE ci_builds.id = ci_job_artifacts.job_id AND ci_job_artifacts.file_type = 3)') + end + end + + def up + queue_background_migration_jobs_by_range_at_intervals( + ::ScheduleToArchiveLegacyTraces::Build.finished.without_archived_trace, + BACKGROUND_MIGRATION_CLASS, + 5.minutes, + batch_size: BATCH_SIZE) + end + + def down + # noop + end +end diff --git a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb index 1a62b30ce81..0765f4149f9 100644 --- a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb +++ b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 20180529152628 do + include TraceHelpers + let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } let(:builds) { table(:ci_builds) } @@ -9,52 +11,31 @@ describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 2 before do namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1') projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123) - build = builds.create!(id: 1, project_id: 123, status: 'success') - - @legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path, - build.created_at.utc.strftime("%Y_%m"), - build.project_id.to_s) - - FileUtils.mkdir_p(@legacy_trace_dir) - - @legacy_trace_path = File.join(@legacy_trace_dir, "#{build.id}.log") + @build = builds.create!(id: 1, project_id: 123, status: 'success', type: 'Ci::Build') end context 'when trace file exsits at the right place' do before do - File.open(@legacy_trace_path, 'wb') { |stream| stream.write('aiueo') } + create_legacy_trace(@build, 'aiueo') end it 'correctly archive legacy traces' do expect(job_artifacts.count).to eq(0) - expect(File.exist?(@legacy_trace_path)).to be_truthy + expect(File.exist?(legacy_trace_path(@build))).to be_truthy described_class.new.perform(1, 1) expect(job_artifacts.count).to eq(1) - expect(File.exist?(@legacy_trace_path)).to be_falsy - expect(File.read(new_trace_path)).to eq('aiueo') + expect(File.exist?(legacy_trace_path(@build))).to be_falsy + expect(File.read(archived_trace_path(job_artifacts.first))).to eq('aiueo') end end context 'when trace file does not exsits at the right place' do - it 'correctly archive legacy traces' do - expect(job_artifacts.count).to eq(0) - expect(File.exist?(@legacy_trace_path)).to be_falsy - + it 'does not raise errors and create job artifact row' do described_class.new.perform(1, 1) expect(job_artifacts.count).to eq(0) end end - - def new_trace_path - job_artifact = job_artifacts.first - - disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s) - creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d') - - File.join(Gitlab.config.artifacts.path, disk_hash[0..1], disk_hash[2..3], disk_hash, - creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s, 'job.log') - end end diff --git a/spec/migrations/archive_legacy_traces_spec.rb b/spec/migrations/archive_legacy_traces_spec.rb deleted file mode 100644 index fc61c4bec17..00000000000 --- a/spec/migrations/archive_legacy_traces_spec.rb +++ /dev/null @@ -1,45 +0,0 @@ -require 'spec_helper' -require Rails.root.join('db', 'post_migrate', '20180529152628_archive_legacy_traces') - -describe ArchiveLegacyTraces, :migration do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:builds) { table(:ci_builds) } - let(:job_artifacts) { table(:ci_job_artifacts) } - - before do - namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1') - projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123) - build = builds.create!(id: 1) - - @legacy_trace_path = File.join( - Settings.gitlab_ci.builds_path, - build.created_at.utc.strftime("%Y_%m"), - build.project_id.to_s, - "#{job.id}.log" - ) - - File.open(@legacy_trace_path, 'wb') { |stream| stream.write('aiueo') } - end - - it 'correctly archive legacy traces' do - expect(job_artifacts.count).to eq(0) - expect(File.exist?(@legacy_trace_path)).to be_truthy - - migrate! - - expect(job_artifacts.count).to eq(1) - expect(File.exist?(@legacy_trace_path)).to be_falsy - expect(File.exist?(new_trace_path)).to be_truthy - end - - def new_trace_path - job_artifact = job_artifacts.first - - disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s) - creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d') - - File.join(disk_hash[0..1], disk_hash[2..3], disk_hash, - creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s) - end -end diff --git a/spec/migrations/schedule_to_archive_legacy_traces_spec.rb b/spec/migrations/schedule_to_archive_legacy_traces_spec.rb new file mode 100644 index 00000000000..d3eac3c45ea --- /dev/null +++ b/spec/migrations/schedule_to_archive_legacy_traces_spec.rb @@ -0,0 +1,45 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20180529152628_schedule_to_archive_legacy_traces') + +describe ScheduleToArchiveLegacyTraces, :migration do + include TraceHelpers + + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:builds) { table(:ci_builds) } + let(:job_artifacts) { table(:ci_job_artifacts) } + + before do + namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123) + @build_success = builds.create!(id: 1, project_id: 123, status: 'success', type: 'Ci::Build') + @build_failed = builds.create!(id: 2, project_id: 123, status: 'failed', type: 'Ci::Build') + @builds_canceled = builds.create!(id: 3, project_id: 123, status: 'canceled', type: 'Ci::Build') + @build_running = builds.create!(id: 4, project_id: 123, status: 'running', type: 'Ci::Build') + + create_legacy_trace(@build_success, 'This job is done') + create_legacy_trace(@build_failed, 'This job is done') + create_legacy_trace(@builds_canceled, 'This job is done') + create_legacy_trace(@build_running, 'This job is not done yet') + end + + it 'correctly archive legacy traces' do + expect(job_artifacts.count).to eq(0) + expect(File.exist?(legacy_trace_path(@build_success))).to be_truthy + expect(File.exist?(legacy_trace_path(@build_failed))).to be_truthy + expect(File.exist?(legacy_trace_path(@builds_canceled))).to be_truthy + expect(File.exist?(legacy_trace_path(@build_running))).to be_truthy + + migrate! + + expect(job_artifacts.count).to eq(3) + expect(File.exist?(legacy_trace_path(@build_success))).to be_falsy + expect(File.exist?(legacy_trace_path(@build_failed))).to be_falsy + expect(File.exist?(legacy_trace_path(@builds_canceled))).to be_falsy + expect(File.exist?(legacy_trace_path(@build_running))).to be_truthy + expect(File.exist?(archived_trace_path(job_artifacts.where(job_id: @build_success.id).first))).to be_truthy + expect(File.exist?(archived_trace_path(job_artifacts.where(job_id: @build_failed.id).first))).to be_truthy + expect(File.exist?(archived_trace_path(job_artifacts.where(job_id: @builds_canceled.id).first))).to be_truthy + expect(job_artifacts.where(job_id: @build_running.id)).not_to be_exist + end +end diff --git a/spec/support/trace/trace_helpers.rb b/spec/support/trace/trace_helpers.rb new file mode 100644 index 00000000000..f6d11b61038 --- /dev/null +++ b/spec/support/trace/trace_helpers.rb @@ -0,0 +1,23 @@ +module TraceHelpers + def create_legacy_trace(build, content) + File.open(legacy_trace_path(build), 'wb') { |stream| stream.write(content) } + end + + def legacy_trace_path(build) + legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path, + build.created_at.utc.strftime("%Y_%m"), + build.project_id.to_s) + + FileUtils.mkdir_p(legacy_trace_dir) + + File.join(legacy_trace_dir, "#{build.id}.log") + end + + def archived_trace_path(job_artifact) + disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s) + creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d') + + File.join(Gitlab.config.artifacts.path, disk_hash[0..1], disk_hash[2..3], disk_hash, + creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s, 'job.log') + end +end -- cgit v1.2.1