From 8c5b3d030210cc2659107ec2633a496165470125 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 28 Mar 2019 20:05:27 +0100 Subject: Allow streaming io objects into Gitaly This allows us to set the encoding of an IO passed without reading it into memory. This is useful if we want to stream files into Gitaly. Like we do when uploading a new file to the repository. --- app/controllers/projects/blob_controller.rb | 3 +-- lib/gitlab/encoding_helper.rb | 7 +++++-- lib/gitlab/gitaly_client/conflicts_service.rb | 2 +- lib/gitlab/gitaly_client/operation_service.rb | 4 ++-- lib/gitlab/gitaly_client/wiki_service.rb | 4 ++-- spec/lib/gitlab/encoding_helper_spec.rb | 13 +++++++++++-- 6 files changed, 22 insertions(+), 11 deletions(-) diff --git a/app/controllers/projects/blob_controller.rb b/app/controllers/projects/blob_controller.rb index 909b17e9c8d..7e072788fc9 100644 --- a/app/controllers/projects/blob_controller.rb +++ b/app/controllers/projects/blob_controller.rb @@ -172,8 +172,7 @@ class Projects::BlobController < Projects::ApplicationController end if params[:file].present? - params[:content] = Base64.encode64(params[:file].read) - params[:encoding] = 'base64' + params[:content] = params[:file] end @commit_params = { diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index a4a154c80f7..5a61a7f5d60 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -76,8 +76,11 @@ module Gitlab str.dup.force_encoding(Encoding::ASCII_8BIT) end - def binary_stringio(str) - StringIO.new(str.freeze || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } + def binary_io(str_or_io) + io = str_or_io.to_io.dup if str_or_io.respond_to?(:to_io) + io ||= StringIO.new(str_or_io.to_s.freeze) + + io.tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } end private diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index 6304f998563..077b63205a8 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -37,7 +37,7 @@ module Gitlab end def resolve_conflicts(target_repository, resolution, source_branch, target_branch) - reader = binary_stringio(resolution.files.to_json) + reader = binary_io(resolution.files.to_json) req_enum = Enumerator.new do |y| header = resolve_conflicts_request_header(target_repository, resolution, source_branch, target_branch) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 2528208440e..b0f328ce3d4 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -294,7 +294,7 @@ module Gitlab action: Gitaly::UserCommitFilesAction.new(header: action_header) ) - reader = binary_stringio(action[:content]) + reader = binary_io(action[:content]) until reader.eof? chunk = reader.read(MAX_MSG_SIZE) @@ -327,7 +327,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, target_branch: encode_binary(branch_name) ) - reader = binary_stringio(patches) + reader = binary_io(patches) chunks = Enumerator.new do |chunk| chunk.yield Gitaly::UserApplyPatchRequest.new(header: header) diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 2b3d622af4d..15c9463e2f2 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -22,7 +22,7 @@ module Gitlab commit_details: gitaly_commit_details(commit_details) ) - strio = binary_stringio(content) + strio = binary_io(content) enum = Enumerator.new do |y| until strio.eof? @@ -49,7 +49,7 @@ module Gitlab commit_details: gitaly_commit_details(commit_details) ) - strio = binary_stringio(content) + strio = binary_io(content) enum = Enumerator.new do |y| until strio.eof? diff --git a/spec/lib/gitlab/encoding_helper_spec.rb b/spec/lib/gitlab/encoding_helper_spec.rb index 429816efec3..88ea98eb1e1 100644 --- a/spec/lib/gitlab/encoding_helper_spec.rb +++ b/spec/lib/gitlab/encoding_helper_spec.rb @@ -189,14 +189,23 @@ describe Gitlab::EncodingHelper do end end - describe '#binary_stringio' do + describe '#binary_io' do it 'does not mutate the original string encoding' do test = 'my-test' - io_stream = ext_class.binary_stringio(test) + io_stream = ext_class.binary_io(test) expect(io_stream.external_encoding.name).to eq('ASCII-8BIT') expect(test.encoding.name).to eq('UTF-8') end + + it 'returns a copy of the IO with the correct encoding' do + test = fixture_file_upload('spec/fixtures/doc_sample.txt').to_io + + io_stream = ext_class.binary_io(test) + + expect(io_stream.external_encoding.name).to eq('ASCII-8BIT') + expect(test).not_to eq(io_stream) + end end end -- cgit v1.2.1