diff options
Diffstat (limited to 'lib/gitlab/ci')
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/build.rb | 3 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/command.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/ci/trace.rb | 32 | ||||
| -rw-r--r-- | lib/gitlab/ci/trace/chunked_io.rb | 231 | ||||
| -rw-r--r-- | lib/gitlab/ci/trace/stream.rb | 11 |
5 files changed, 265 insertions, 14 deletions
diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 70732d26bbd..b5eb0cfa2f0 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -14,7 +14,8 @@ module Gitlab trigger_requests: Array(@command.trigger_request), user: @command.current_user, pipeline_schedule: @command.schedule, - protected: @command.protected_ref? + protected: @command.protected_ref?, + variables_attributes: Array(@command.variables_attributes) ) @pipeline.set_config_source diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index a1849b01c5d..a53c80d34f7 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -7,7 +7,7 @@ module Gitlab # rubocop:disable Naming/FileName :origin_ref, :checkout_sha, :after_sha, :before_sha, :trigger_request, :schedule, :ignore_skip_ci, :save_incompleted, - :seeds_block + :seeds_block, :variables_attributes ) do include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 47b67930c6d..fe15fabc2e8 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -36,16 +36,16 @@ module Gitlab end def set(data) - write do |stream| + write('w+b') do |stream| data = job.hide_secrets(data) stream.set(data) end end def append(data, offset) - write do |stream| + write('a+b') do |stream| current_length = stream.size - break -current_length unless current_length == offset + break current_length unless current_length == offset data = job.hide_secrets(data) stream.append(data, offset) @@ -54,13 +54,15 @@ module Gitlab end def exist? - trace_artifact&.exists? || current_path.present? || old_trace.present? + trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present? end def read stream = Gitlab::Ci::Trace::Stream.new do if trace_artifact trace_artifact.open + elsif job.trace_chunks.any? + Gitlab::Ci::Trace::ChunkedIO.new(job) elsif current_path File.open(current_path, "rb") elsif old_trace @@ -73,9 +75,15 @@ module Gitlab stream&.close end - def write + def write(mode) stream = Gitlab::Ci::Trace::Stream.new do - File.open(ensure_path, "a+b") + if current_path + File.open(current_path, mode) + elsif Feature.enabled?('ci_enable_live_trace') + Gitlab::Ci::Trace::ChunkedIO.new(job) + else + File.open(ensure_path, mode) + end end yield(stream).tap do @@ -92,6 +100,7 @@ module Gitlab FileUtils.rm(trace_path, force: true) end + job.trace_chunks.fast_destroy_all job.erase_old_trace! end @@ -99,7 +108,12 @@ module Gitlab raise ArchiveError, 'Already archived' if trace_artifact raise ArchiveError, 'Job is not finished yet' unless job.complete? - if current_path + if job.trace_chunks.any? + Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| + archive_stream!(stream) + stream.destroy! + end + elsif current_path File.open(current_path) do |stream| archive_stream!(stream) FileUtils.rm(current_path) @@ -116,7 +130,7 @@ module Gitlab def archive_stream!(stream) clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path| - create_job_trace!(job, clone_path) + create_build_trace!(job, clone_path) end end @@ -132,7 +146,7 @@ module Gitlab end end - def create_job_trace!(job, path) + def create_build_trace!(job, path) File.open(path) do |stream| job.create_job_artifacts_trace!( project: job.project, diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb new file mode 100644 index 00000000000..bfe0c2a2c26 --- /dev/null +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -0,0 +1,231 @@ +## +# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) +# source: https://gitlab.com/snippets/1685610 +module Gitlab + module Ci + class Trace + class ChunkedIO + CHUNK_SIZE = ::Ci::BuildTraceChunk::CHUNK_SIZE + + FailedToGetChunkError = Class.new(StandardError) + + attr_reader :build + attr_reader :tell, :size + attr_reader :chunk, :chunk_range + + alias_method :pos, :tell + + def initialize(build, &block) + @build = build + @chunks_cache = [] + @tell = 0 + @size = calculate_size + yield self if block_given? + end + + def close + # no-op + end + + def binmode + # no-op + end + + def binmode? + true + end + + def seek(pos, where = IO::SEEK_SET) + new_pos = + case where + when IO::SEEK_END + size + pos + when IO::SEEK_SET + pos + when IO::SEEK_CUR + tell + pos + else + -1 + end + + raise ArgumentError, 'new position is outside of file' if new_pos < 0 || new_pos > size + + @tell = new_pos + end + + def eof? + tell == size + end + + def each_line + until eof? + line = readline + break if line.nil? + + yield(line) + end + end + + def read(length = nil, outbuf = "") + out = "" + + length ||= size - tell + + until length <= 0 || eof? + data = chunk_slice_from_offset + break if data.empty? + + chunk_bytes = [CHUNK_SIZE - chunk_offset, length].min + chunk_data = data.byteslice(0, chunk_bytes) + + out << chunk_data + @tell += chunk_data.bytesize + length -= chunk_data.bytesize + end + + # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality + if outbuf + outbuf.slice!(0, outbuf.bytesize) + outbuf << out + end + + out + end + + def readline + out = "" + + until eof? + data = chunk_slice_from_offset + new_line = data.index("\n") + + if !new_line.nil? + out << data[0..new_line] + @tell += new_line + 1 + break + else + out << data + @tell += data.bytesize + end + end + + out + end + + def write(data) + start_pos = tell + + while tell < start_pos + data.bytesize + # get slice from current offset till the end where it falls into chunk + chunk_bytes = CHUNK_SIZE - chunk_offset + chunk_data = data.byteslice(tell - start_pos, chunk_bytes) + + # append data to chunk, overwriting from that point + ensure_chunk.append(chunk_data, chunk_offset) + + # move offsets within buffer + @tell += chunk_data.bytesize + @size = [size, tell].max + end + + tell - start_pos + ensure + invalidate_chunk_cache + end + + def truncate(offset) + raise ArgumentError, 'Outside of file' if offset > size || offset < 0 + return if offset == size # Skip the following process as it doesn't affect anything + + @tell = offset + @size = offset + + # remove all next chunks + trace_chunks.where('chunk_index > ?', chunk_index).fast_destroy_all + + # truncate current chunk + current_chunk.truncate(chunk_offset) + ensure + invalidate_chunk_cache + end + + def flush + # no-op + end + + def present? + true + end + + def destroy! + trace_chunks.fast_destroy_all + @tell = @size = 0 + ensure + invalidate_chunk_cache + end + + private + + ## + # The below methods are not implemented in IO class + # + def in_range? + @chunk_range&.include?(tell) + end + + def chunk_slice_from_offset + unless in_range? + current_chunk.tap do |chunk| + raise FailedToGetChunkError unless chunk + + @chunk = chunk.data + @chunk_range = chunk.range + end + end + + @chunk[chunk_offset..CHUNK_SIZE] + end + + def chunk_offset + tell % CHUNK_SIZE + end + + def chunk_index + tell / CHUNK_SIZE + end + + def chunk_start + chunk_index * CHUNK_SIZE + end + + def chunk_end + [chunk_start + CHUNK_SIZE, size].min + end + + def invalidate_chunk_cache + @chunks_cache = [] + end + + def current_chunk + @chunks_cache[chunk_index] ||= trace_chunks.find_by(chunk_index: chunk_index) + end + + def build_chunk + @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index) + end + + def ensure_chunk + current_chunk || build_chunk + end + + def trace_chunks + ::Ci::BuildTraceChunk.where(build: build) + end + + def calculate_size + trace_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 187ad8b833a..a71040e5e56 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -39,6 +39,8 @@ module Gitlab end def append(data, offset) + data = data.force_encoding(Encoding::BINARY) + stream.truncate(offset) stream.seek(0, IO::SEEK_END) stream.write(data) @@ -46,8 +48,11 @@ module Gitlab end def set(data) - truncate(0) + data = data.force_encoding(Encoding::BINARY) + + stream.seek(0, IO::SEEK_SET) stream.write(data) + stream.truncate(data.bytesize) stream.flush() end @@ -127,11 +132,11 @@ module Gitlab buf += debris debris, *lines = buf.each_line.to_a lines.reverse_each do |line| - yield(line.force_encoding('UTF-8')) + yield(line.force_encoding(Encoding.default_external)) end end - yield(debris.force_encoding('UTF-8')) unless debris.empty? + yield(debris.force_encoding(Encoding.default_external)) unless debris.empty? end def read_backward(length) |