diff options
| -rw-r--r-- | db/post_migrate/20180529152628_archive_legacy_traces.rb | 44 | ||||
| -rw-r--r-- | db/schema.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/background_migration/archive_legacy_traces.rb | 79 | ||||
| -rw-r--r-- | spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb | 60 | ||||
| -rw-r--r-- | spec/migrations/archive_legacy_traces_spec.rb | 45 |
5 files changed, 229 insertions, 1 deletions
diff --git a/db/post_migrate/20180529152628_archive_legacy_traces.rb b/db/post_migrate/20180529152628_archive_legacy_traces.rb new file mode 100644 index 00000000000..78ec1ab1a94 --- /dev/null +++ b/db/post_migrate/20180529152628_archive_legacy_traces.rb @@ -0,0 +1,44 @@ +class ArchiveLegacyTraces < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 10_000 + BACKGROUND_MIGRATION_CLASS = 'ArchiveLegacyTraces' + + disable_ddl_transaction! + + class Build < ActiveRecord::Base + include EachBatch + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled # Disable STI + + scope :finished, -> { where(status: [:success, :failed, :canceled]) } + + scope :without_new_traces, ->() do + where('NOT EXISTS (?)', + ::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) + end + end + + class JobArtifact < ActiveRecord::Base + self.table_name = 'ci_job_artifacts' + + enum file_type: { + archive: 1, + metadata: 2, + trace: 3 + } + end + + def up + queue_background_migration_jobs_by_range_at_intervals( + ::ArchiveLegacyTraces::Build.finished.without_new_traces, + BACKGROUND_MIGRATION_CLASS, + 5.minutes, + batch_size: BATCH_SIZE) + end + + def down + # noop + end +end diff --git a/db/schema.rb b/db/schema.rb index 97247387bc7..a8f8e14a3fc 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180529093006) do +ActiveRecord::Schema.define(version: 20180529152628) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb new file mode 100644 index 00000000000..9741a7c181e --- /dev/null +++ b/lib/gitlab/background_migration/archive_legacy_traces.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class ArchiveLegacyTraces + class Build < ActiveRecord::Base + include ::HasStatus + + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled # Disable STI + + belongs_to :project, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Project' + has_one :job_artifacts_trace, -> () { where(file_type: ArchiveLegacyTraces::JobArtifact.file_types[:trace]) }, class_name: 'ArchiveLegacyTraces::JobArtifact', foreign_key: :job_id + has_many :trace_chunks, foreign_key: :build_id, class_name: 'ArchiveLegacyTraces::BuildTraceChunk' + + scope :finished, -> { where(status: [:success, :failed, :canceled]) } + + scope :without_new_traces, ->() do + finished.where('NOT EXISTS (?)', + BackgroundMigration::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) + end + + def trace + ::Gitlab::Ci::Trace.new(self) + end + + def trace=(data) + raise NotImplementedError + end + + def old_trace + read_attribute(:trace) + end + + def erase_old_trace! + update_column(:trace, nil) + end + end + + class JobArtifact < ActiveRecord::Base + self.table_name = 'ci_job_artifacts' + + belongs_to :build + belongs_to :project + + mount_uploader :file, JobArtifactUploader + + enum file_type: { + archive: 1, + metadata: 2, + trace: 3 + } + end + + class BuildTraceChunk < ActiveRecord::Base + self.table_name = 'ci_build_trace_chunks' + + belongs_to :build + end + + class Project < ActiveRecord::Base + self.table_name = 'projects' + + has_many :builds, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Build' + end + + def perform(start_id, stop_id) + BackgroundMigration::ArchiveLegacyTraces::Build + .finished + .without_new_traces + .where(id: (start_id..stop_id)).find_each do |build| + build.trace.archive! + end + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb new file mode 100644 index 00000000000..ecc6eea9284 --- /dev/null +++ b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb @@ -0,0 +1,60 @@ +require 'spec_helper' + +describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 20180529152628 do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:builds) { table(:ci_builds) } + let(:job_artifacts) { table(:ci_job_artifacts) } + + before do + namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123) + build = builds.create!(id: 1, project_id: 123, status: 'success') + + @legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path, + build.created_at.utc.strftime("%Y_%m"), + build.project_id.to_s) + + FileUtils.mkdir_p(@legacy_trace_dir) + + @legacy_trace_path = File.join(@legacy_trace_dir, "#{build.id}.log") + end + + context 'when trace file exsits at the right place' do + before do + File.open(@legacy_trace_path, 'wb') { |stream| stream.write('aiueo') } + end + + it 'correctly archive legacy traces' do + expect(job_artifacts.count).to eq(0) + expect(File.exist?(@legacy_trace_path)).to be_truthy + + described_class.new.perform(1, 1) + + expect(job_artifacts.count).to eq(1) + expect(File.exist?(@legacy_trace_path)).to be_falsy + expect(File.read(new_trace_path)).to eq('aiueo') + end + end + + context 'when trace file does not exsits at the right place' do + it 'correctly archive legacy traces' do + expect(job_artifacts.count).to eq(0) + expect(File.exist?(@legacy_trace_path)).to be_falsy + + described_class.new.perform(1, 1) + + expect(job_artifacts.count).to eq(0) + end + end + + def new_trace_path + job_artifact = job_artifacts.first + + disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s) + creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d') + + File.join(Gitlab.config.artifacts.path, disk_hash[0..1], disk_hash[2..3], disk_hash, + creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s, 'job.log') + end +end diff --git a/spec/migrations/archive_legacy_traces_spec.rb b/spec/migrations/archive_legacy_traces_spec.rb new file mode 100644 index 00000000000..fc61c4bec17 --- /dev/null +++ b/spec/migrations/archive_legacy_traces_spec.rb @@ -0,0 +1,45 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20180529152628_archive_legacy_traces') + +describe ArchiveLegacyTraces, :migration do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:builds) { table(:ci_builds) } + let(:job_artifacts) { table(:ci_job_artifacts) } + + before do + namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123) + build = builds.create!(id: 1) + + @legacy_trace_path = File.join( + Settings.gitlab_ci.builds_path, + build.created_at.utc.strftime("%Y_%m"), + build.project_id.to_s, + "#{job.id}.log" + ) + + File.open(@legacy_trace_path, 'wb') { |stream| stream.write('aiueo') } + end + + it 'correctly archive legacy traces' do + expect(job_artifacts.count).to eq(0) + expect(File.exist?(@legacy_trace_path)).to be_truthy + + migrate! + + expect(job_artifacts.count).to eq(1) + expect(File.exist?(@legacy_trace_path)).to be_falsy + expect(File.exist?(new_trace_path)).to be_truthy + end + + def new_trace_path + job_artifact = job_artifacts.first + + disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s) + creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d') + + File.join(disk_hash[0..1], disk_hash[2..3], disk_hash, + creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s) + end +end |