diff options
18 files changed, 149 insertions, 150 deletions
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 56216093293..61a0299d4fb 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -19,14 +19,13 @@ module Ci has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment' has_many :trace_sections, class_name: 'Ci::BuildTraceSection' + has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id - has_many :chunks, class_name: 'Ci::JobTraceChunk', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_one :metadata, class_name: 'Ci::BuildMetadata' delegate :timeout, to: :metadata, prefix: true, allow_nil: true delegate :gitlab_deploy_token, to: :project diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 47302265fb7..f3beb6d4156 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -1,8 +1,8 @@ module Ci - class JobTraceChunk < ActiveRecord::Base + class BuildTraceChunk < ActiveRecord::Base extend Gitlab::Ci::Model - belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id after_destroy :redis_delete_data, if: :redis? @@ -118,11 +118,11 @@ module Ci end def redis_data_key - "gitlab:ci:trace:#{job_id}:chunks:#{chunk_index}:data" + "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}:data" end def redis_lock_key - "gitlab:ci:trace:#{job_id}:chunks:#{chunk_index}:lock" + "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}:lock" end def in_lock diff --git a/app/workers/build_trace_swap_chunk_worker.rb b/app/workers/build_trace_swap_chunk_worker.rb index 29b7cd4f808..1392b9d34e1 100644 --- a/app/workers/build_trace_swap_chunk_worker.rb +++ b/app/workers/build_trace_swap_chunk_worker.rb @@ -4,9 +4,9 @@ class BuildTraceSwapChunkWorker queue_namespace :pipeline_processing - def perform(job_trace_chunk_id) - Ci::JobTraceChunk.find_by(id: job_trace_chunk_id).try do |job_trace_chunk| - job_trace_chunk.use_database! + def perform(build_trace_chunk_id) + Ci::BuildTraceChunk.find_by(id: build_trace_chunk_id).try do |build_trace_chunk| + build_trace_chunk.use_database! end end end diff --git a/db/migrate/20180326202229_create_ci_build_trace_chunks.rb b/db/migrate/20180326202229_create_ci_build_trace_chunks.rb new file mode 100644 index 00000000000..fb3f5786e85 --- /dev/null +++ b/db/migrate/20180326202229_create_ci_build_trace_chunks.rb @@ -0,0 +1,17 @@ +class CreateCiBuildTraceChunks < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + create_table :ci_build_trace_chunks, id: :bigserial do |t| + t.integer :build_id, null: false + t.integer :chunk_index, null: false + t.integer :data_store, null: false + t.binary :raw_data + + t.foreign_key :ci_builds, column: :build_id, on_delete: :cascade + t.index [:build_id, :chunk_index], unique: true + end + end +end diff --git a/db/migrate/20180326202229_create_ci_job_trace_chunks.rb b/db/migrate/20180326202229_create_ci_job_trace_chunks.rb deleted file mode 100644 index 23bc478eb29..00000000000 --- a/db/migrate/20180326202229_create_ci_job_trace_chunks.rb +++ /dev/null @@ -1,17 +0,0 @@ -class CreateCiJobTraceChunks < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - def change - create_table :ci_job_trace_chunks, id: :bigserial do |t| - t.integer :job_id, null: false - t.integer :chunk_index, null: false - t.integer :data_store, null: false - t.binary :raw_data - - t.foreign_key :ci_builds, column: :job_id, on_delete: :cascade - t.index [:job_id, :chunk_index], unique: true - end - end -end diff --git a/db/migrate/20180406204716_add_limits_ci_job_trace_chunks_raw_data_for_mysql.rb b/db/migrate/20180406204716_add_limits_ci_build_trace_chunks_raw_data_for_mysql.rb index e7343db7da0..0f2734853e6 100644 --- a/db/migrate/20180406204716_add_limits_ci_job_trace_chunks_raw_data_for_mysql.rb +++ b/db/migrate/20180406204716_add_limits_ci_build_trace_chunks_raw_data_for_mysql.rb @@ -1,13 +1,13 @@ # See http://doc.gitlab.com/ce/development/migration_style_guide.html # for more information on how to write migrations for GitLab. -require Rails.root.join('db/migrate/limits_ci_job_trace_chunks_raw_data_for_mysql') +require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql') -class AddLimitsCiJobTraceChunksRawDataForMysql < ActiveRecord::Migration +class AddLimitsCiBuildTraceChunksRawDataForMysql < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers DOWNTIME = false def up - LimitsCiJobTraceChunksRawDataForMysql.new.up + LimitsCiBuildTraceChunksRawDataForMysql.new.up end end diff --git a/db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql.rb b/db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql.rb new file mode 100644 index 00000000000..e1771912c3c --- /dev/null +++ b/db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql.rb @@ -0,0 +1,9 @@ +class LimitsCiBuildTraceChunksRawDataForMysql < ActiveRecord::Migration + def up + return unless Gitlab::Database.mysql? + + # Mysql needs MEDIUMTEXT type (up to 16MB) rather than TEXT (up to 64KB) + # Because 'raw_data' is always capped by Ci::BuildTraceChunk::CHUNK_SIZE, which is 128KB + change_column :ci_build_trace_chunks, :raw_data, :binary, limit: 16.megabytes - 1 #MEDIUMTEXT + end +end diff --git a/db/migrate/limits_ci_job_trace_chunks_raw_data_for_mysql.rb b/db/migrate/limits_ci_job_trace_chunks_raw_data_for_mysql.rb deleted file mode 100644 index 38310b7443a..00000000000 --- a/db/migrate/limits_ci_job_trace_chunks_raw_data_for_mysql.rb +++ /dev/null @@ -1,9 +0,0 @@ -class LimitsCiJobTraceChunksRawDataForMysql < ActiveRecord::Migration - def up - return unless Gitlab::Database.mysql? - - # Mysql needs MEDIUMTEXT type (up to 16MB) rather than TEXT (up to 64KB) - # Because 'raw_data' is always capped by Ci::JobTraceChunk::CHUNK_SIZE, which is 128KB - change_column :ci_job_trace_chunks, :raw_data, :binary, limit: 16.megabytes - 1 #MEDIUMTEXT - end -end diff --git a/db/schema.rb b/db/schema.rb index 19ae9cbf443..06232f68894 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180425131009) do +ActiveRecord::Schema.define(version: 20180425205249) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -246,6 +246,15 @@ ActiveRecord::Schema.define(version: 20180425131009) do add_index "chat_teams", ["namespace_id"], name: "index_chat_teams_on_namespace_id", unique: true, using: :btree + create_table "ci_build_trace_chunks", id: :bigserial, force: :cascade do |t| + t.integer "build_id", null: false + t.integer "chunk_index", null: false + t.integer "data_store", null: false + t.binary "raw_data" + end + + add_index "ci_build_trace_chunks", ["build_id", "chunk_index"], name: "index_ci_build_trace_chunks_on_build_id_and_chunk_index", unique: true, using: :btree + create_table "ci_build_trace_section_names", force: :cascade do |t| t.integer "project_id", null: false t.string "name", null: false @@ -371,15 +380,6 @@ ActiveRecord::Schema.define(version: 20180425131009) do add_index "ci_job_artifacts", ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true, using: :btree add_index "ci_job_artifacts", ["project_id"], name: "index_ci_job_artifacts_on_project_id", using: :btree - create_table "ci_job_trace_chunks", id: :bigserial, force: :cascade do |t| - t.integer "job_id", null: false - t.integer "chunk_index", null: false - t.integer "data_store", null: false - t.binary "raw_data" - end - - add_index "ci_job_trace_chunks", ["job_id", "chunk_index"], name: "index_ci_job_trace_chunks_on_job_id_and_chunk_index", unique: true, using: :btree - create_table "ci_pipeline_schedule_variables", force: :cascade do |t| t.string "key", null: false t.text "value" @@ -2075,6 +2075,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do add_foreign_key "boards", "namespaces", column: "group_id", on_delete: :cascade add_foreign_key "boards", "projects", name: "fk_f15266b5f9", on_delete: :cascade add_foreign_key "chat_teams", "namespaces", on_delete: :cascade + add_foreign_key "ci_build_trace_chunks", "ci_builds", column: "build_id", on_delete: :cascade add_foreign_key "ci_build_trace_section_names", "projects", on_delete: :cascade add_foreign_key "ci_build_trace_sections", "ci_build_trace_section_names", column: "section_name_id", name: "fk_264e112c66", on_delete: :cascade add_foreign_key "ci_build_trace_sections", "ci_builds", column: "build_id", name: "fk_4ebe41f502", on_delete: :cascade @@ -2087,7 +2088,6 @@ ActiveRecord::Schema.define(version: 20180425131009) do add_foreign_key "ci_group_variables", "namespaces", column: "group_id", name: "fk_33ae4d58d8", on_delete: :cascade add_foreign_key "ci_job_artifacts", "ci_builds", column: "job_id", on_delete: :cascade add_foreign_key "ci_job_artifacts", "projects", on_delete: :cascade - add_foreign_key "ci_job_trace_chunks", "ci_builds", column: "job_id", on_delete: :cascade add_foreign_key "ci_pipeline_schedule_variables", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_41c35fda51", on_delete: :cascade add_foreign_key "ci_pipeline_schedules", "projects", name: "fk_8ead60fcc4", on_delete: :cascade add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 79838cabdb5..65c8b9118c6 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -54,14 +54,14 @@ module Gitlab end def exist? - trace_artifact&.exists? || job.chunks.any? || current_path.present? || old_trace.present? + trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present? end def read stream = Gitlab::Ci::Trace::Stream.new do if trace_artifact trace_artifact.open - elsif job.chunks.any? + elsif job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) elsif current_path File.open(current_path, "rb") @@ -100,7 +100,7 @@ module Gitlab FileUtils.rm(trace_path, force: true) end - job.chunks.destroy_all + job.trace_chunks.destroy_all job.erase_old_trace! end @@ -108,7 +108,7 @@ module Gitlab raise ArchiveError, 'Already archived' if trace_artifact raise ArchiveError, 'Job is not finished yet' unless job.complete? - if job.chunks.any? + if job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| archive_stream!(stream) stream.destroy! @@ -130,7 +130,7 @@ module Gitlab def archive_stream!(stream) clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path| - create_job_trace!(job, clone_path) + create_build_trace!(job, clone_path) end end @@ -146,7 +146,7 @@ module Gitlab end end - def create_job_trace!(job, path) + def create_build_trace!(job, path) File.open(path) do |stream| job.create_job_artifacts_trace!( project: job.project, diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 6b4a9f61961..877be08b219 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -5,18 +5,18 @@ module Gitlab module Ci class Trace class ChunkedIO - CHUNK_SIZE = ::Ci::JobTraceChunk::CHUNK_SIZE + CHUNK_SIZE = ::Ci::BuildTraceChunk::CHUNK_SIZE FailedToGetChunkError = Class.new(StandardError) - attr_reader :job + attr_reader :build attr_reader :tell, :size attr_reader :chunk, :chunk_range alias_method :pos, :tell - def initialize(job, &block) - @job = job + def initialize(build, &block) + @build = build @chunks_cache = [] @tell = 0 @size = calculate_size @@ -140,7 +140,7 @@ module Gitlab @size = offset # remove all next chunks - job_chunks.where('chunk_index > ?', chunk_index).destroy_all + trace_chunks.where('chunk_index > ?', chunk_index).destroy_all # truncate current chunk current_chunk.truncate(chunk_offset) if chunk_offset != 0 @@ -157,7 +157,7 @@ module Gitlab end def destroy! - job_chunks.destroy_all + trace_chunks.destroy_all @tell = @size = 0 ensure invalidate_chunk_cache @@ -206,23 +206,23 @@ module Gitlab end def current_chunk - @chunks_cache[chunk_index] ||= job_chunks.find_by(chunk_index: chunk_index) + @chunks_cache[chunk_index] ||= trace_chunks.find_by(chunk_index: chunk_index) end def build_chunk - @chunks_cache[chunk_index] = ::Ci::JobTraceChunk.new(job: job, chunk_index: chunk_index) + @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index) end def ensure_chunk current_chunk || build_chunk end - def job_chunks - ::Ci::JobTraceChunk.where(job: job) + def trace_chunks + ::Ci::BuildTraceChunk.where(build: build) end def calculate_size - job_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i + trace_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i end end end diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake index 3cdcdcdf874..c6204f89de4 100644 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ b/lib/tasks/migrate/add_limits_mysql.rake @@ -1,7 +1,7 @@ require Rails.root.join('db/migrate/limits_to_mysql') require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql') -require Rails.root.join('db/migrate/limits_ci_job_trace_chunks_raw_data_for_mysql') +require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql') desc "GitLab | Add limits to strings in mysql database" task add_limits_mysql: :environment do @@ -9,5 +9,5 @@ task add_limits_mysql: :environment do LimitsToMysql.new.up MarkdownCacheLimitsToMysql.new.up MergeRequestDiffFileLimitsToMysql.new.up - LimitsCiJobTraceChunksRawDataForMysql.new.up + LimitsCiBuildTraceChunksRawDataForMysql.new.up end diff --git a/spec/factories/ci/job_trace_chunks.rb b/spec/factories/ci/build_trace_chunks.rb index e2cc2e77dda..be13a84a47c 100644 --- a/spec/factories/ci/job_trace_chunks.rb +++ b/spec/factories/ci/build_trace_chunks.rb @@ -1,5 +1,5 @@ FactoryBot.define do - factory :ci_job_trace_chunk, class: Ci::JobTraceChunk do + factory :ci_build_trace_chunk, class: Ci::BuildTraceChunk do job factory: :ci_build chunk_index 0 data_store :redis diff --git a/spec/lib/gitlab/ci/trace/chunked_io_spec.rb b/spec/lib/gitlab/ci/trace/chunked_io_spec.rb index bcef21d5f71..6259b952add 100644 --- a/spec/lib/gitlab/ci/trace/chunked_io_spec.rb +++ b/spec/lib/gitlab/ci/trace/chunked_io_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do include ChunkedIOHelpers - set(:job) { create(:ci_build, :running) } - let(:chunked_io) { described_class.new(job) } + set(:build) { create(:ci_build, :running) } + let(:chunked_io) { described_class.new(build) } before do stub_feature_flags(ci_enable_live_trace: true) @@ -13,7 +13,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context "#initialize" do context 'when a chunk exists' do before do - job.trace.set('ABC') + build.trace.set('ABC') end it { expect(chunked_io.size).to eq(3) } @@ -22,7 +22,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when two chunks exist' do before do stub_buffer_size(4) - job.trace.set('ABCDEF') + build.trace.set('ABCDEF') end it { expect(chunked_io.size).to eq(6) } @@ -37,7 +37,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do subject { chunked_io.seek(pos, where) } before do - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end context 'when moves pos to end of the file' do @@ -68,7 +68,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do subject { chunked_io.eof? } before do - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end context 'when current pos is at end of the file' do @@ -94,7 +94,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is smaller than file size' do before do stub_buffer_size(sample_trace_raw.bytesize / 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it 'yields lines' do @@ -106,7 +106,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is larger than file size' do before do stub_buffer_size(sample_trace_raw.bytesize * 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it 'calls get_chunk only once' do @@ -127,7 +127,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is smaller than file size' do before do stub_buffer_size(sample_trace_raw.bytesize / 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it { is_expected.to eq(sample_trace_raw) } @@ -136,7 +136,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is larger than file size' do before do stub_buffer_size(sample_trace_raw.bytesize * 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it { is_expected.to eq(sample_trace_raw) } @@ -149,7 +149,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is smaller than file size' do before do stub_buffer_size(sample_trace_raw.bytesize / 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it 'reads a trace' do @@ -160,7 +160,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is larger than file size' do before do stub_buffer_size(sample_trace_raw.bytesize * 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it 'reads a trace' do @@ -175,7 +175,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is smaller than file size' do before do stub_buffer_size(sample_trace_raw.bytesize / 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it 'reads a trace' do @@ -186,7 +186,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is larger than file size' do before do stub_buffer_size(sample_trace_raw.bytesize * 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it 'reads a trace' do @@ -201,7 +201,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is smaller than file size' do before do stub_buffer_size(sample_trace_raw.bytesize / 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it 'reads a trace' do @@ -212,7 +212,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is larger than file size' do before do stub_buffer_size(sample_trace_raw.bytesize * 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it 'reads a trace' do @@ -238,7 +238,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is smaller than file size' do before do stub_buffer_size(sample_trace_raw.bytesize / 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it_behaves_like 'all line matching' @@ -247,7 +247,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is larger than file size' do before do stub_buffer_size(sample_trace_raw.bytesize * 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it_behaves_like 'all line matching' @@ -256,7 +256,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when pos is at middle of the file' do before do stub_buffer_size(sample_trace_raw.bytesize / 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) chunked_io.seek(chunked_io.size / 2) string_io.seek(string_io.size / 2) @@ -316,7 +316,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is smaller than file size' do before do stub_buffer_size(sample_trace_raw.bytesize / 2) - job.trace.set(exist_data) + build.trace.set(exist_data) end it_behaves_like 'appends a trace' @@ -325,7 +325,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is larger than file size' do before do stub_buffer_size(sample_trace_raw.bytesize * 2) - job.trace.set(exist_data) + build.trace.set(exist_data) end it_behaves_like 'appends a trace' @@ -349,7 +349,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is smaller than file size' do before do stub_buffer_size(sample_trace_raw.bytesize / 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it_behaves_like 'truncates a trace' @@ -358,7 +358,7 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do context 'when buffer size is larger than file size' do before do stub_buffer_size(sample_trace_raw.bytesize * 2) - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it_behaves_like 'truncates a trace' @@ -370,14 +370,14 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do subject { chunked_io.destroy! } before do - job.trace.set(sample_trace_raw) + build.trace.set(sample_trace_raw) end it 'deletes' do expect { subject }.to change { chunked_io.size } .from(sample_trace_raw.bytesize).to(0) - expect(Ci::JobTraceChunk.where(job: job).count).to eq(0) + expect(Ci::BuildTraceChunk.where(build: build).count).to eq(0) end end end diff --git a/spec/lib/gitlab/ci/trace/stream_spec.rb b/spec/lib/gitlab/ci/trace/stream_spec.rb index 7b6f5bf83ff..4f49958dd33 100644 --- a/spec/lib/gitlab/ci/trace/stream_spec.rb +++ b/spec/lib/gitlab/ci/trace/stream_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do - set(:job) { create(:ci_build, :running) } + set(:build) { create(:ci_build, :running) } before do stub_feature_flags(ci_enable_live_trace: true) @@ -83,7 +83,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do context 'when stream is ChunkedIO' do let(:stream) do described_class.new do - Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io| + Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io| chunked_io.write((1..8).to_a.join("\n")) chunked_io.seek(0, IO::SEEK_SET) end @@ -137,7 +137,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do context 'when stream is ChunkedIO' do let(:stream) do described_class.new do - Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io| + Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io| chunked_io.write('12345678') chunked_io.seek(0, IO::SEEK_SET) end @@ -175,7 +175,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do context 'when stream is ChunkedIO' do let(:stream) do described_class.new do - Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io| + Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io| chunked_io.write('12345678') chunked_io.seek(0, IO::SEEK_SET) end @@ -234,7 +234,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do context 'when stream is ChunkedIO' do let(:stream) do described_class.new do - Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io| + Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io| chunked_io.write(File.binread(path)) chunked_io.seek(0, IO::SEEK_SET) end @@ -283,7 +283,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do context 'when stream is ChunkedIO' do let(:stream) do described_class.new do - Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io| + Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io| chunked_io.write("1234") chunked_io.seek(0, IO::SEEK_SET) end @@ -318,7 +318,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do context 'when stream is ChunkedIO' do let(:stream) do described_class.new do - Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io| + Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io| chunked_io.write("12\n34\n56") chunked_io.seek(0, IO::SEEK_SET) end @@ -473,7 +473,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do context 'when stream is ChunkedIO' do let(:stream) do described_class.new do - Gitlab::Ci::Trace::ChunkedIO.new(job).tap do |chunked_io| + Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io| chunked_io.write(data) chunked_io.seek(0, IO::SEEK_SET) end diff --git a/spec/lib/gitlab/ci/trace_spec.rb b/spec/lib/gitlab/ci/trace_spec.rb index 4d80b58889b..faa23461a91 100644 --- a/spec/lib/gitlab/ci/trace_spec.rb +++ b/spec/lib/gitlab/ci/trace_spec.rb @@ -436,7 +436,7 @@ describe Gitlab::Ci::Trace, :clean_gitlab_redis_cache do it "can be erased" do trace.erase! expect(trace.exist?).to be(false) - expect(Ci::JobTraceChunk.where(job: build)).not_to be_exist + expect(Ci::BuildTraceChunk.where(build: build)).not_to be_exist end it "returns live trace data" do @@ -512,7 +512,7 @@ describe Gitlab::Ci::Trace, :clean_gitlab_redis_cache do expect(build.trace.exist?).to be_truthy expect(build.job_artifacts_trace.file.exists?).to be_truthy expect(build.job_artifacts_trace.file.filename).to eq('job.log') - expect(Ci::JobTraceChunk.where(job: build)).not_to be_exist + expect(Ci::BuildTraceChunk.where(build: build)).not_to be_exist expect(src_checksum) .to eq(Digest::SHA256.file(build.job_artifacts_trace.file.path).hexdigest) expect(build.job_artifacts_trace.file_sha256).to eq(src_checksum) diff --git a/spec/models/ci/job_trace_chunk_spec.rb b/spec/models/ci/build_trace_chunk_spec.rb index b01dfd097e6..988e23146d2 100644 --- a/spec/models/ci/job_trace_chunk_spec.rb +++ b/spec/models/ci/build_trace_chunk_spec.rb @@ -1,13 +1,13 @@ require 'spec_helper' -describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do - set(:job) { create(:ci_build, :running) } +describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do + set(:build) { create(:ci_build, :running) } let(:chunk_index) { 0 } let(:data_store) { :redis } let(:raw_data) { nil } - - let(:job_trace_chunk) do - described_class.new(job: job, chunk_index: chunk_index, data_store: data_store, raw_data: raw_data) + + let(:build_trace_chunk) do + described_class.new(build: build, chunk_index: chunk_index, data_store: data_store, raw_data: raw_data) end describe 'CHUNK_SIZE' do @@ -17,13 +17,13 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do end describe '#data' do - subject { job_trace_chunk.data } + subject { build_trace_chunk.data } context 'when data_store is redis' do let(:data_store) { :redis } before do - job_trace_chunk.send(:redis_set_data, 'Sample data in redis') + build_trace_chunk.send(:redis_set_data, 'Sample data in redis') end it { is_expected.to eq('Sample data in redis') } @@ -38,7 +38,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do context 'when data_store is others' do before do - job_trace_chunk.send(:write_attribute, :data_store, -1) + build_trace_chunk.send(:write_attribute, :data_store, -1) end it { expect { subject }.to raise_error('Unsupported data store') } @@ -46,7 +46,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do end describe '#set_data' do - subject { job_trace_chunk.set_data(value) } + subject { build_trace_chunk.set_data(value) } let(:value) { 'Sample data' } @@ -60,11 +60,11 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do let(:data_store) { :redis } it do - expect(job_trace_chunk.send(:redis_data)).to be_nil + expect(build_trace_chunk.send(:redis_data)).to be_nil subject - expect(job_trace_chunk.send(:redis_data)).to eq(value) + expect(build_trace_chunk.send(:redis_data)).to eq(value) end context 'when fullfilled chunk size' do @@ -82,26 +82,26 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do let(:data_store) { :db } it 'sets data' do - expect(job_trace_chunk.raw_data).to be_nil + expect(build_trace_chunk.raw_data).to be_nil subject - expect(job_trace_chunk.raw_data).to eq(value) - expect(job_trace_chunk.persisted?).to be_truthy + expect(build_trace_chunk.raw_data).to eq(value) + expect(build_trace_chunk.persisted?).to be_truthy end context 'when raw_data is not changed' do it 'does not execute UPDATE' do - expect(job_trace_chunk.raw_data).to be_nil - job_trace_chunk.save! + expect(build_trace_chunk.raw_data).to be_nil + build_trace_chunk.save! # First set expect(ActiveRecord::QueryRecorder.new { subject }.count).to be > 0 - expect(job_trace_chunk.raw_data).to eq(value) - expect(job_trace_chunk.persisted?).to be_truthy + expect(build_trace_chunk.raw_data).to eq(value) + expect(build_trace_chunk.persisted?).to be_truthy # Second set - job_trace_chunk.reload + build_trace_chunk.reload expect(ActiveRecord::QueryRecorder.new { subject }.count).to be(0) end end @@ -117,7 +117,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do context 'when data_store is others' do before do - job_trace_chunk.send(:write_attribute, :data_store, -1) + build_trace_chunk.send(:write_attribute, :data_store, -1) end it { expect { subject }.to raise_error('Unsupported data store') } @@ -125,7 +125,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do end describe '#truncate' do - subject { job_trace_chunk.truncate(offset) } + subject { build_trace_chunk.truncate(offset) } shared_examples_for 'truncates' do context 'when offset is negative' do @@ -146,7 +146,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do it 'truncates' do subject - expect(job_trace_chunk.data).to eq(data.byteslice(0, offset)) + expect(build_trace_chunk.data).to eq(data.byteslice(0, offset)) end end end @@ -156,7 +156,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do let(:data) { 'Sample data in redis' } before do - job_trace_chunk.send(:redis_set_data, data) + build_trace_chunk.send(:redis_set_data, data) end it_behaves_like 'truncates' @@ -172,7 +172,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do end describe '#append' do - subject { job_trace_chunk.append(new_data, offset) } + subject { build_trace_chunk.append(new_data, offset) } let(:new_data) { 'Sample new data' } let(:offset) { 0 } @@ -203,7 +203,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do it 'appends' do subject - expect(job_trace_chunk.data).to eq(total_data) + expect(build_trace_chunk.data).to eq(total_data) end end @@ -213,7 +213,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do it 'appends' do subject - expect(job_trace_chunk.data).to eq(data.byteslice(0, offset) + new_data) + expect(build_trace_chunk.data).to eq(data.byteslice(0, offset) + new_data) end end end @@ -223,7 +223,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do let(:data) { 'Sample data in redis' } before do - job_trace_chunk.send(:redis_set_data, data) + build_trace_chunk.send(:redis_set_data, data) end it_behaves_like 'appends' @@ -239,7 +239,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do end describe '#size' do - subject { job_trace_chunk.size } + subject { build_trace_chunk.size } context 'when data_store is redis' do let(:data_store) { :redis } @@ -248,7 +248,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do let(:data) { 'Sample data in redis' } before do - job_trace_chunk.send(:redis_set_data, data) + build_trace_chunk.send(:redis_set_data, data) end it { is_expected.to eq(data.bytesize) } @@ -276,7 +276,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do end describe '#use_database!' do - subject { job_trace_chunk.use_database! } + subject { build_trace_chunk.use_database! } context 'when data_store is redis' do let(:data_store) { :redis } @@ -285,19 +285,19 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do let(:data) { 'Sample data in redis' } before do - job_trace_chunk.send(:redis_set_data, data) + build_trace_chunk.send(:redis_set_data, data) end it 'stashes the data' do - expect(job_trace_chunk.data_store).to eq('redis') - expect(job_trace_chunk.send(:redis_data)).to eq(data) - expect(job_trace_chunk.raw_data).to be_nil + expect(build_trace_chunk.data_store).to eq('redis') + expect(build_trace_chunk.send(:redis_data)).to eq(data) + expect(build_trace_chunk.raw_data).to be_nil subject - expect(job_trace_chunk.data_store).to eq('db') - expect(job_trace_chunk.send(:redis_data)).to be_nil - expect(job_trace_chunk.raw_data).to eq(data) + expect(build_trace_chunk.data_store).to eq('db') + expect(build_trace_chunk.send(:redis_data)).to be_nil + expect(build_trace_chunk.raw_data).to eq(data) end end @@ -320,11 +320,11 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do describe 'ExclusiveLock' do before do allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) { nil } - stub_const('Ci::JobTraceChunk::LOCK_RETRY', 1) + stub_const('Ci::BuildTraceChunk::LOCK_RETRY', 1) end it 'raise an error' do - expect { job_trace_chunk.append('ABC', 0) }.to raise_error('Failed to obtain write lock') + expect { build_trace_chunk.append('ABC', 0) }.to raise_error('Failed to obtain write lock') end end @@ -338,7 +338,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do create(:ci_build, :running, :trace_live, pipeline: pipeline, project: project) end - shared_examples_for 'deletes all job_trace_chunk and data in redis' do + shared_examples_for 'deletes all build_trace_chunk and data in redis' do it do project.builds.each do |build| Gitlab::Redis::SharedState.with do |redis| @@ -364,20 +364,20 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do end end - context 'when job_trace_chunk is destroyed' do + context 'when build_trace_chunk is destroyed' do let(:subject) do project.builds.each { |build| build.chunks.destroy_all } end - it_behaves_like 'deletes all job_trace_chunk and data in redis' + it_behaves_like 'deletes all build_trace_chunk and data in redis' end - context 'when job is destroyed' do + context 'when build is destroyed' do let(:subject) do project.builds.destroy_all end - it_behaves_like 'deletes all job_trace_chunk and data in redis' + it_behaves_like 'deletes all build_trace_chunk and data in redis' end context 'when project is destroyed' do @@ -385,7 +385,7 @@ describe Ci::JobTraceChunk, :clean_gitlab_redis_shared_state do project.destroy! end - it_behaves_like 'deletes all job_trace_chunk and data in redis' + it_behaves_like 'deletes all build_trace_chunk and data in redis' end end end diff --git a/spec/support/chunked_io/chunked_io_helpers.rb b/spec/support/chunked_io/chunked_io_helpers.rb index 57391e6d42a..4238a4b3e94 100644 --- a/spec/support/chunked_io/chunked_io_helpers.rb +++ b/spec/support/chunked_io/chunked_io_helpers.rb @@ -5,7 +5,7 @@ module ChunkedIOHelpers end def stub_buffer_size(size) - stub_const('Ci::JobTraceChunk::CHUNK_SIZE', size) + stub_const('Ci::BuildTraceChunk::CHUNK_SIZE', size) stub_const('Gitlab::Ci::Trace::ChunkedIO::CHUNK_SIZE', size) end |