diff options
Diffstat (limited to 'spec')
41 files changed, 2065 insertions, 86 deletions
diff --git a/spec/controllers/concerns/send_file_upload_spec.rb b/spec/controllers/concerns/send_file_upload_spec.rb new file mode 100644 index 00000000000..f4c99ea4064 --- /dev/null +++ b/spec/controllers/concerns/send_file_upload_spec.rb @@ -0,0 +1,89 @@ +require 'spec_helper' + +describe SendFileUpload do + let(:uploader_class) do + Class.new(GitlabUploader) do + include ObjectStorage::Concern + + storage_options Gitlab.config.uploads + + private + + # user/:id + def dynamic_segment + File.join(model.class.to_s.underscore, model.id.to_s) + end + end + end + + let(:controller_class) do + Class.new do + include SendFileUpload + end + end + + let(:object) { build_stubbed(:user) } + let(:uploader) { uploader_class.new(object, :file) } + + describe '#send_upload' do + let(:controller) { controller_class.new } + let(:temp_file) { Tempfile.new('test') } + + subject { controller.send_upload(uploader) } + + before do + FileUtils.touch(temp_file) + end + + after do + FileUtils.rm_f(temp_file) + end + + context 'when local file is used' do + before do + uploader.store!(temp_file) + end + + it 'sends a file' do + expect(controller).to receive(:send_file).with(uploader.path, anything) + + subject + end + end + + context 'when remote file is used' do + before do + stub_uploads_object_storage(uploader: uploader_class) + uploader.object_store = ObjectStorage::Store::REMOTE + uploader.store!(temp_file) + end + + context 'and proxying is enabled' do + before do + allow(Gitlab.config.uploads.object_store).to receive(:proxy_download) { true } + end + + it 'sends a file' do + headers = double + expect(headers).to receive(:store).with(Gitlab::Workhorse::SEND_DATA_HEADER, /^send-url:/) + expect(controller).to receive(:headers) { headers } + expect(controller).to receive(:head).with(:ok) + + subject + end + end + + context 'and proxying is disabled' do + before do + allow(Gitlab.config.uploads.object_store).to receive(:proxy_download) { false } + end + + it 'sends a file' do + expect(controller).to receive(:redirect_to).with(/#{uploader.path}/) + + subject + end + end + end + end +end diff --git a/spec/controllers/projects/artifacts_controller_spec.rb b/spec/controllers/projects/artifacts_controller_spec.rb index 25a2e13fe1a..4ea6f869aa3 100644 --- a/spec/controllers/projects/artifacts_controller_spec.rb +++ b/spec/controllers/projects/artifacts_controller_spec.rb @@ -145,9 +145,23 @@ describe Projects::ArtifactsController do context 'when using local file storage' do it_behaves_like 'a valid file' do let(:job) { create(:ci_build, :success, :artifacts, pipeline: pipeline) } + let(:store) { ObjectStorage::Store::LOCAL } let(:archive_path) { JobArtifactUploader.root } end end + + context 'when using remote file storage' do + before do + stub_artifacts_object_storage + end + + it_behaves_like 'a valid file' do + let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) } + let!(:job) { create(:ci_build, :success, pipeline: pipeline) } + let(:store) { ObjectStorage::Store::REMOTE } + let(:archive_path) { 'https://' } + end + end end end diff --git a/spec/controllers/projects/jobs_controller_spec.rb b/spec/controllers/projects/jobs_controller_spec.rb index f3e303bb0fe..31046c202e6 100644 --- a/spec/controllers/projects/jobs_controller_spec.rb +++ b/spec/controllers/projects/jobs_controller_spec.rb @@ -1,7 +1,9 @@ +# coding: utf-8 require 'spec_helper' describe Projects::JobsController do include ApiHelpers + include HttpIOHelpers let(:project) { create(:project, :public) } let(:pipeline) { create(:ci_pipeline, project: project) } @@ -203,6 +205,41 @@ describe Projects::JobsController do end end + context 'when trace artifact is in ObjectStorage' do + let!(:job) { create(:ci_build, :success, :trace_artifact, pipeline: pipeline) } + + before do + allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false } + allow_any_instance_of(JobArtifactUploader).to receive(:url) { remote_trace_url } + allow_any_instance_of(JobArtifactUploader).to receive(:size) { remote_trace_size } + end + + context 'when there are no network issues' do + before do + stub_remote_trace_206 + + get_trace + end + + it 'returns a trace' do + expect(response).to have_gitlab_http_status(:ok) + expect(json_response['id']).to eq job.id + expect(json_response['status']).to eq job.status + expect(json_response['html']).to eq(job.trace.html) + end + end + + context 'when there is a network issue' do + before do + stub_remote_trace_500 + end + + it 'returns a trace' do + expect { get_trace }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError) + end + end + end + def get_trace get :trace, namespace_id: project.namespace, project_id: project, @@ -446,14 +483,18 @@ describe Projects::JobsController do end describe 'GET raw' do - before do - get_raw + subject do + post :raw, namespace_id: project.namespace, + project_id: project, + id: job.id end context 'when job has a trace artifact' do let(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) } it 'returns a trace' do + response = subject + expect(response).to have_gitlab_http_status(:ok) expect(response.content_type).to eq 'text/plain; charset=utf-8' expect(response.body).to eq job.job_artifacts_trace.open.read @@ -464,6 +505,8 @@ describe Projects::JobsController do let(:job) { create(:ci_build, :trace_live, pipeline: pipeline) } it 'send a trace file' do + response = subject + expect(response).to have_gitlab_http_status(:ok) expect(response.content_type).to eq 'text/plain; charset=utf-8' expect(response.body).to eq 'BUILD TRACE' @@ -474,14 +517,22 @@ describe Projects::JobsController do let(:job) { create(:ci_build, pipeline: pipeline) } it 'returns not_found' do + response = subject + expect(response).to have_gitlab_http_status(:not_found) end end - def get_raw - post :raw, namespace_id: project.namespace, - project_id: project, - id: job.id + context 'when the trace artifact is in ObjectStorage' do + let!(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) } + + before do + allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false } + end + + it 'redirect to the trace file url' do + expect(subject).to redirect_to(job.job_artifacts_trace.file.url) + end end end end diff --git a/spec/controllers/projects/raw_controller_spec.rb b/spec/controllers/projects/raw_controller_spec.rb index b7df42168e0..08e2ccf893a 100644 --- a/spec/controllers/projects/raw_controller_spec.rb +++ b/spec/controllers/projects/raw_controller_spec.rb @@ -8,10 +8,7 @@ describe Projects::RawController do let(:id) { 'master/README.md' } it 'delivers ASCII file' do - get(:show, - namespace_id: public_project.namespace.to_param, - project_id: public_project, - id: id) + get_show(public_project, id) expect(response).to have_gitlab_http_status(200) expect(response.header['Content-Type']).to eq('text/plain; charset=utf-8') @@ -25,10 +22,7 @@ describe Projects::RawController do let(:id) { 'master/files/images/6049019_460s.jpg' } it 'sets image content type header' do - get(:show, - namespace_id: public_project.namespace.to_param, - project_id: public_project, - id: id) + get_show(public_project, id) expect(response).to have_gitlab_http_status(200) expect(response.header['Content-Type']).to eq('image/jpeg') @@ -54,21 +48,40 @@ describe Projects::RawController do it 'serves the file' do expect(controller).to receive(:send_file).with("#{LfsObjectUploader.root}/91/ef/f75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897", filename: 'lfs_object.iso', disposition: 'attachment') - get(:show, - namespace_id: public_project.namespace.to_param, - project_id: public_project, - id: id) + get_show(public_project, id) expect(response).to have_gitlab_http_status(200) end + + context 'and lfs uses object storage' do + before do + lfs_object.file = fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png") + lfs_object.save! + stub_lfs_object_storage + lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) + end + + it 'responds with redirect to file' do + get_show(public_project, id) + + expect(response).to have_gitlab_http_status(302) + expect(response.location).to include(lfs_object.reload.file.path) + end + + it 'sets content disposition' do + get_show(public_project, id) + + file_uri = URI.parse(response.location) + params = CGI.parse(file_uri.query) + + expect(params["response-content-disposition"].first).to eq 'attachment;filename="lfs_object.iso"' + end + end end context 'when project does not have access' do it 'does not serve the file' do - get(:show, - namespace_id: public_project.namespace.to_param, - project_id: public_project, - id: id) + get_show(public_project, id) expect(response).to have_gitlab_http_status(404) end @@ -81,10 +94,7 @@ describe Projects::RawController do end it 'delivers ASCII file' do - get(:show, - namespace_id: public_project.namespace.to_param, - project_id: public_project, - id: id) + get_show(public_project, id) expect(response).to have_gitlab_http_status(200) expect(response.header['Content-Type']).to eq('text/plain; charset=utf-8') @@ -95,4 +105,10 @@ describe Projects::RawController do end end end + + def get_show(project, id) + get(:show, namespace_id: project.namespace.to_param, + project_id: project, + id: id) + end end diff --git a/spec/factories/appearances.rb b/spec/factories/appearances.rb index 5f9c57c0c8d..18c7453bd1b 100644 --- a/spec/factories/appearances.rb +++ b/spec/factories/appearances.rb @@ -2,8 +2,21 @@ FactoryBot.define do factory :appearance do - title "MepMep" - description "This is my Community Edition instance" + title "GitLab Community Edition" + description "Open source software to collaborate on code" new_project_guidelines "Custom project guidelines" end + + trait :with_logo do + logo { fixture_file_upload('spec/fixtures/dk.png') } + end + + trait :with_header_logo do + header_logo { fixture_file_upload('spec/fixtures/dk.png') } + end + + trait :with_logos do + with_logo + with_header_logo + end end diff --git a/spec/factories/ci/job_artifacts.rb b/spec/factories/ci/job_artifacts.rb index 8544d54ccaa..3d3287d8168 100644 --- a/spec/factories/ci/job_artifacts.rb +++ b/spec/factories/ci/job_artifacts.rb @@ -5,6 +5,10 @@ FactoryBot.define do job factory: :ci_build file_type :archive + trait :remote_store do + file_store JobArtifactUploader::Store::REMOTE + end + after :build do |artifact| artifact.project ||= artifact.job.project end diff --git a/spec/factories/lfs_objects.rb b/spec/factories/lfs_objects.rb index caaed4d5246..eaf3a4ed497 100644 --- a/spec/factories/lfs_objects.rb +++ b/spec/factories/lfs_objects.rb @@ -15,4 +15,8 @@ FactoryBot.define do trait :correct_oid do oid 'b804383982bb89b00e828e3f44c038cc991d3d1768009fc39ba8e2c081b9fb75' end + + trait :object_storage do + file_store { LfsObjectUploader::Store::REMOTE } + end end diff --git a/spec/factories/uploads.rb b/spec/factories/uploads.rb index ff3a2a76acc..b45f6f30e40 100644 --- a/spec/factories/uploads.rb +++ b/spec/factories/uploads.rb @@ -5,6 +5,7 @@ FactoryBot.define do uploader "AvatarUploader" mount_point :avatar secret nil + store ObjectStorage::Store::LOCAL # we should build a mount agnostic upload by default transient do @@ -27,6 +28,10 @@ FactoryBot.define do secret SecureRandom.hex end + trait :object_storage do + store ObjectStorage::Store::REMOTE + end + trait :namespace_upload do model { build(:group) } path { File.join(secret, filename) } diff --git a/spec/initializers/fog_google_https_private_urls_spec.rb b/spec/initializers/fog_google_https_private_urls_spec.rb new file mode 100644 index 00000000000..de3c157ab7b --- /dev/null +++ b/spec/initializers/fog_google_https_private_urls_spec.rb @@ -0,0 +1,24 @@ +require 'spec_helper' + +describe 'Fog::Storage::GoogleXML::File' do + let(:storage) do + Fog.mock! + Fog::Storage.new({ + google_storage_access_key_id: "asdf", + google_storage_secret_access_key: "asdf", + provider: "Google" + }) + end + + let(:file) do + directory = storage.directories.create(key: 'data') + directory.files.create( + body: 'Hello World!', + key: 'hello_world.txt' + ) + end + + it 'delegates to #get_https_url' do + expect(file.url(Time.now)).to start_with("https://") + end +end diff --git a/spec/lib/gitlab/ci/trace/http_io_spec.rb b/spec/lib/gitlab/ci/trace/http_io_spec.rb new file mode 100644 index 00000000000..5474e2f518c --- /dev/null +++ b/spec/lib/gitlab/ci/trace/http_io_spec.rb @@ -0,0 +1,315 @@ +require 'spec_helper' + +describe Gitlab::Ci::Trace::HttpIO do + include HttpIOHelpers + + let(:http_io) { described_class.new(url, size) } + let(:url) { remote_trace_url } + let(:size) { remote_trace_size } + + describe '#close' do + subject { http_io.close } + + it { is_expected.to be_nil } + end + + describe '#binmode' do + subject { http_io.binmode } + + it { is_expected.to be_nil } + end + + describe '#binmode?' do + subject { http_io.binmode? } + + it { is_expected.to be_truthy } + end + + describe '#path' do + subject { http_io.path } + + it { is_expected.to be_nil } + end + + describe '#url' do + subject { http_io.url } + + it { is_expected.to eq(url) } + end + + describe '#seek' do + subject { http_io.seek(pos, where) } + + context 'when moves pos to end of the file' do + let(:pos) { 0 } + let(:where) { IO::SEEK_END } + + it { is_expected.to eq(size) } + end + + context 'when moves pos to middle of the file' do + let(:pos) { size / 2 } + let(:where) { IO::SEEK_SET } + + it { is_expected.to eq(size / 2) } + end + + context 'when moves pos around' do + it 'matches the result' do + expect(http_io.seek(0)).to eq(0) + expect(http_io.seek(100, IO::SEEK_CUR)).to eq(100) + expect { http_io.seek(size + 1, IO::SEEK_CUR) }.to raise_error('new position is outside of file') + end + end + end + + describe '#eof?' do + subject { http_io.eof? } + + context 'when current pos is at end of the file' do + before do + http_io.seek(size, IO::SEEK_SET) + end + + it { is_expected.to be_truthy } + end + + context 'when current pos is not at end of the file' do + before do + http_io.seek(0, IO::SEEK_SET) + end + + it { is_expected.to be_falsey } + end + end + + describe '#each_line' do + subject { http_io.each_line } + + let(:string_io) { StringIO.new(remote_trace_body) } + + before do + stub_remote_trace_206 + end + + it 'yields lines' do + expect { |b| http_io.each_line(&b) }.to yield_successive_args(*string_io.each_line.to_a) + end + + context 'when buckets on GCS' do + context 'when BUFFER_SIZE is larger than file size' do + before do + stub_remote_trace_200 + set_larger_buffer_size_than(size) + end + + it 'calls get_chunk only once' do + expect_any_instance_of(Net::HTTP).to receive(:request).once.and_call_original + + http_io.each_line { |line| } + end + end + end + end + + describe '#read' do + subject { http_io.read(length) } + + context 'when there are no network issue' do + before do + stub_remote_trace_206 + end + + context 'when read whole size' do + let(:length) { nil } + + context 'when BUFFER_SIZE is smaller than file size' do + before do + set_smaller_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to eq(remote_trace_body) + end + end + + context 'when BUFFER_SIZE is larger than file size' do + before do + set_larger_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to eq(remote_trace_body) + end + end + end + + context 'when read only first 100 bytes' do + let(:length) { 100 } + + context 'when BUFFER_SIZE is smaller than file size' do + before do + set_smaller_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to eq(remote_trace_body[0, length]) + end + end + + context 'when BUFFER_SIZE is larger than file size' do + before do + set_larger_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to eq(remote_trace_body[0, length]) + end + end + end + + context 'when tries to read oversize' do + let(:length) { size + 1000 } + + context 'when BUFFER_SIZE is smaller than file size' do + before do + set_smaller_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to eq(remote_trace_body) + end + end + + context 'when BUFFER_SIZE is larger than file size' do + before do + set_larger_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to eq(remote_trace_body) + end + end + end + + context 'when tries to read 0 bytes' do + let(:length) { 0 } + + context 'when BUFFER_SIZE is smaller than file size' do + before do + set_smaller_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to be_empty + end + end + + context 'when BUFFER_SIZE is larger than file size' do + before do + set_larger_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to be_empty + end + end + end + end + + context 'when there is anetwork issue' do + let(:length) { nil } + + before do + stub_remote_trace_500 + end + + it 'reads a trace' do + expect { subject }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError) + end + end + end + + describe '#readline' do + subject { http_io.readline } + + let(:string_io) { StringIO.new(remote_trace_body) } + + before do + stub_remote_trace_206 + end + + shared_examples 'all line matching' do + it 'reads a line' do + (0...remote_trace_body.lines.count).each do + expect(http_io.readline).to eq(string_io.readline) + end + end + end + + context 'when there is anetwork issue' do + let(:length) { nil } + + before do + stub_remote_trace_500 + end + + it 'reads a trace' do + expect { subject }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError) + end + end + + context 'when BUFFER_SIZE is smaller than file size' do + before do + set_smaller_buffer_size_than(size) + end + + it_behaves_like 'all line matching' + end + + context 'when BUFFER_SIZE is larger than file size' do + before do + set_larger_buffer_size_than(size) + end + + it_behaves_like 'all line matching' + end + + context 'when pos is at middle of the file' do + before do + set_smaller_buffer_size_than(size) + + http_io.seek(size / 2) + string_io.seek(size / 2) + end + + it 'reads from pos' do + expect(http_io.readline).to eq(string_io.readline) + end + end + end + + describe '#write' do + subject { http_io.write(nil) } + + it { expect { subject }.to raise_error(NotImplementedError) } + end + + describe '#truncate' do + subject { http_io.truncate(nil) } + + it { expect { subject }.to raise_error(NotImplementedError) } + end + + describe '#flush' do + subject { http_io.flush } + + it { expect { subject }.to raise_error(NotImplementedError) } + end + + describe '#present?' do + subject { http_io.present? } + + it { is_expected.to be_truthy } + end +end diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 44e4c6ff94b..0716852f57f 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -265,7 +265,9 @@ CommitStatus: - target_url - description - artifacts_file +- artifacts_file_store - artifacts_metadata +- artifacts_metadata_store - erased_by_id - erased_at - artifacts_expire_at diff --git a/spec/lib/gitlab/verify/lfs_objects_spec.rb b/spec/lib/gitlab/verify/lfs_objects_spec.rb index 64f3a9660e0..0f890e2c7ce 100644 --- a/spec/lib/gitlab/verify/lfs_objects_spec.rb +++ b/spec/lib/gitlab/verify/lfs_objects_spec.rb @@ -31,5 +31,21 @@ describe Gitlab::Verify::LfsObjects do expect(failures.keys).to contain_exactly(lfs_object) expect(failure.to_s).to include('Checksum mismatch') end + + context 'with remote files' do + before do + stub_lfs_object_storage + end + + it 'skips LFS objects in object storage' do + local_failure = create(:lfs_object) + create(:lfs_object, :object_storage) + + failures = {} + described_class.new(batch_size: 10).run_batches { |_, failed| failures.merge!(failed) } + + expect(failures.keys).to contain_exactly(local_failure) + end + end end end diff --git a/spec/lib/gitlab/verify/uploads_spec.rb b/spec/lib/gitlab/verify/uploads_spec.rb index 6146ce61226..85768308edc 100644 --- a/spec/lib/gitlab/verify/uploads_spec.rb +++ b/spec/lib/gitlab/verify/uploads_spec.rb @@ -40,5 +40,21 @@ describe Gitlab::Verify::Uploads do expect(failures.keys).to contain_exactly(upload) expect(failure.to_s).to include('Checksum missing') end + + context 'with remote files' do + before do + stub_uploads_object_storage(AvatarUploader) + end + + it 'skips uploads in object storage' do + local_failure = create(:upload) + create(:upload, :object_storage) + + failures = {} + described_class.new(batch_size: 10).run_batches { |_, failed| failures.merge!(failed) } + + expect(failures.keys).to contain_exactly(local_failure) + end + end end end diff --git a/spec/migrations/remove_empty_fork_networks_spec.rb b/spec/migrations/remove_empty_fork_networks_spec.rb index 7f7ce91378b..f6d030ab25c 100644 --- a/spec/migrations/remove_empty_fork_networks_spec.rb +++ b/spec/migrations/remove_empty_fork_networks_spec.rb @@ -19,6 +19,10 @@ describe RemoveEmptyForkNetworks, :migration do deleted_project.destroy! end + after do + Upload.reset_column_information + end + it 'deletes only the fork network without members' do expect(fork_networks.count).to eq(2) diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 30a352fd090..7d935cf8d76 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -198,6 +198,16 @@ describe Ci::Build do end context 'when legacy artifacts are used' do + let(:build) { create(:ci_build, :legacy_artifacts) } + + subject { build.artifacts? } + + context 'is expired' do + let(:build) { create(:ci_build, :legacy_artifacts, :expired) } + + it { is_expected.to be_falsy } + end + context 'artifacts archive does not exist' do let(:build) { create(:ci_build) } @@ -208,13 +218,25 @@ describe Ci::Build do let(:build) { create(:ci_build, :legacy_artifacts) } it { is_expected.to be_truthy } + end + end + end - context 'is expired' do - let(:build) { create(:ci_build, :legacy_artifacts, :expired) } + describe '#browsable_artifacts?' do + subject { build.browsable_artifacts? } - it { is_expected.to be_falsy } - end + context 'artifacts metadata does not exist' do + before do + build.update_attributes(legacy_artifacts_metadata: nil) end + + it { is_expected.to be_falsy } + end + + context 'artifacts metadata does exists' do + let(:build) { create(:ci_build, :artifacts) } + + it { is_expected.to be_truthy } end end diff --git a/spec/models/ci/job_artifact_spec.rb b/spec/models/ci/job_artifact_spec.rb index a2bd36537e6..1aa28434879 100644 --- a/spec/models/ci/job_artifact_spec.rb +++ b/spec/models/ci/job_artifact_spec.rb @@ -15,6 +15,50 @@ describe Ci::JobArtifact do it { is_expected.to delegate_method(:open).to(:file) } it { is_expected.to delegate_method(:exists?).to(:file) } + describe 'callbacks' do + subject { create(:ci_job_artifact, :archive) } + + describe '#schedule_background_upload' do + context 'when object storage is disabled' do + before do + stub_artifacts_object_storage(enabled: false) + end + + it 'does not schedule the migration' do + expect(ObjectStorageUploadWorker).not_to receive(:perform_async) + + subject + end + end + + context 'when object storage is enabled' do + context 'when background upload is enabled' do + before do + stub_artifacts_object_storage(background_upload: true) + end + + it 'schedules the model for migration' do + expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with('JobArtifactUploader', described_class.name, :file, kind_of(Numeric)) + + subject + end + end + + context 'when background upload is disabled' do + before do + stub_artifacts_object_storage(background_upload: false) + end + + it 'schedules the model for migration' do + expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async) + + subject + end + end + end + end + end + describe '#set_size' do it 'sets the size' do expect(artifact.size).to eq(106365) diff --git a/spec/models/lfs_object_spec.rb b/spec/models/lfs_object_spec.rb new file mode 100644 index 00000000000..a182116d637 --- /dev/null +++ b/spec/models/lfs_object_spec.rb @@ -0,0 +1,85 @@ +require 'spec_helper' + +describe LfsObject do + describe '#local_store?' do + it 'returns true when file_store is nil' do + subject.file_store = nil + + expect(subject.local_store?).to eq true + end + + it 'returns true when file_store is equal to LfsObjectUploader::Store::LOCAL' do + subject.file_store = LfsObjectUploader::Store::LOCAL + + expect(subject.local_store?).to eq true + end + + it 'returns false whe file_store is equal to LfsObjectUploader::Store::REMOTE' do + subject.file_store = LfsObjectUploader::Store::REMOTE + + expect(subject.local_store?).to eq false + end + end + + describe '#schedule_background_upload' do + before do + stub_lfs_setting(enabled: true) + end + + subject { create(:lfs_object, :with_file) } + + context 'when object storage is disabled' do + before do + stub_lfs_object_storage(enabled: false) + end + + it 'does not schedule the migration' do + expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async) + + subject + end + end + + context 'when object storage is enabled' do + context 'when background upload is enabled' do + context 'when is licensed' do + before do + stub_lfs_object_storage(background_upload: true) + end + + it 'schedules the model for migration' do + expect(ObjectStorage::BackgroundMoveWorker) + .to receive(:perform_async) + .with('LfsObjectUploader', described_class.name, :file, kind_of(Numeric)) + .once + + subject + end + + it 'schedules the model for migration once' do + expect(ObjectStorage::BackgroundMoveWorker) + .to receive(:perform_async) + .with('LfsObjectUploader', described_class.name, :file, kind_of(Numeric)) + .once + + lfs_object = create(:lfs_object) + lfs_object.file = fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png") + lfs_object.save! + end + end + end + + context 'when background upload is disabled' do + before do + stub_lfs_object_storage(background_upload: false) + end + + it 'schedules the model for migration' do + expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async) + + subject + end + end + end + end +end diff --git a/spec/requests/api/jobs_spec.rb b/spec/requests/api/jobs_spec.rb index 6192bbd4abb..3ffdfdc0e9a 100644 --- a/spec/requests/api/jobs_spec.rb +++ b/spec/requests/api/jobs_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' describe API::Jobs do + include HttpIOHelpers + set(:project) do create(:project, :repository, public_builds: false) end @@ -112,6 +114,7 @@ describe API::Jobs do let(:query) { Hash.new } before do + job get api("/projects/#{project.id}/pipelines/#{pipeline.id}/jobs", api_user), query end @@ -335,10 +338,55 @@ describe API::Jobs do end end + context 'when artifacts are stored remotely' do + let(:proxy_download) { false } + + before do + stub_artifacts_object_storage(proxy_download: proxy_download) + end + + let(:job) { create(:ci_build, pipeline: pipeline) } + let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) } + + before do + job.reload + + get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user) + end + + context 'when proxy download is enabled' do + let(:proxy_download) { true } + + it 'responds with the workhorse send-url' do + expect(response.headers[Gitlab::Workhorse::SEND_DATA_HEADER]).to start_with("send-url:") + end + end + + context 'when proxy download is disabled' do + it 'returns location redirect' do + expect(response).to have_gitlab_http_status(302) + end + end + + context 'authorized user' do + it 'returns the file remote URL' do + expect(response).to redirect_to(artifact.file.url) + end + end + + context 'unauthorized user' do + let(:api_user) { nil } + + it 'does not return specific job artifacts' do + expect(response).to have_gitlab_http_status(404) + end + end + end + it 'does not return job artifacts if not uploaded' do get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user) - expect(response).to have_gitlab_http_status(404) + expect(response).to have_gitlab_http_status(:not_found) end end end @@ -349,6 +397,7 @@ describe API::Jobs do let(:job) { create(:ci_build, :artifacts, pipeline: pipeline, user: api_user) } before do + stub_artifacts_object_storage job.success end @@ -412,9 +461,24 @@ describe API::Jobs do "attachment; filename=#{job.artifacts_file.filename}" } end - it { expect(response).to have_gitlab_http_status(200) } + it { expect(response).to have_http_status(:ok) } it { expect(response.headers).to include(download_headers) } end + + context 'when artifacts are stored remotely' do + let(:job) { create(:ci_build, pipeline: pipeline, user: api_user) } + let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) } + + before do + job.reload + + get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user) + end + + it 'returns location redirect' do + expect(response).to have_http_status(:found) + end + end end context 'with regular branch' do @@ -451,6 +515,22 @@ describe API::Jobs do end context 'authorized user' do + context 'when trace is in ObjectStorage' do + let!(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) } + + before do + stub_remote_trace_206 + allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false } + allow_any_instance_of(JobArtifactUploader).to receive(:url) { remote_trace_url } + allow_any_instance_of(JobArtifactUploader).to receive(:size) { remote_trace_size } + end + + it 'returns specific job trace' do + expect(response).to have_gitlab_http_status(200) + expect(response.body).to eq(job.trace.raw) + end + end + context 'when trace is artifact' do let(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) } diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb index 95c23726a79..f3dd121faa9 100644 --- a/spec/requests/api/runner_spec.rb +++ b/spec/requests/api/runner_spec.rb @@ -200,7 +200,7 @@ describe API::Runner do let(:project) { create(:project, shared_runners_enabled: false) } let(:pipeline) { create(:ci_pipeline_without_jobs, project: project, ref: 'master') } let(:runner) { create(:ci_runner) } - let!(:job) do + let(:job) do create(:ci_build, :artifacts, :extended_options, pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0, commands: "ls\ndate") end @@ -215,6 +215,7 @@ describe API::Runner do let(:user_agent) { 'gitlab-runner 9.0.0 (9-0-stable; go1.7.4; linux/amd64)' } before do + job stub_container_registry_config(enabled: false) end @@ -888,6 +889,7 @@ describe API::Runner do let(:file_upload2) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/gif') } before do + stub_artifacts_object_storage job.run! end @@ -1179,27 +1181,67 @@ describe API::Runner do describe 'GET /api/v4/jobs/:id/artifacts' do let(:token) { job.token } - before do - download_artifact - end - context 'when job has artifacts' do - let(:job) { create(:ci_build, :artifacts) } - let(:download_headers) do - { 'Content-Transfer-Encoding' => 'binary', - 'Content-Disposition' => 'attachment; filename=ci_build_artifacts.zip' } + let(:job) { create(:ci_build) } + let(:store) { JobArtifactUploader::Store::LOCAL } + + before do + create(:ci_job_artifact, :archive, file_store: store, job: job) end context 'when using job token' do - it 'download artifacts' do - expect(response).to have_gitlab_http_status(200) - expect(response.headers).to include download_headers + context 'when artifacts are stored locally' do + let(:download_headers) do + { 'Content-Transfer-Encoding' => 'binary', + 'Content-Disposition' => 'attachment; filename=ci_build_artifacts.zip' } + end + + before do + download_artifact + end + + it 'download artifacts' do + expect(response).to have_http_status(200) + expect(response.headers).to include download_headers + end + end + + context 'when artifacts are stored remotely' do + let(:store) { JobArtifactUploader::Store::REMOTE } + let!(:job) { create(:ci_build) } + + context 'when proxy download is being used' do + before do + download_artifact(direct_download: false) + end + + it 'uses workhorse send-url' do + expect(response).to have_gitlab_http_status(200) + expect(response.headers).to include( + 'Gitlab-Workhorse-Send-Data' => /send-url:/) + end + end + + context 'when direct download is being used' do + before do + download_artifact(direct_download: true) + end + + it 'receive redirect for downloading artifacts' do + expect(response).to have_gitlab_http_status(302) + expect(response.headers).to include('Location') + end + end end end context 'when using runnners token' do let(:token) { job.project.runners_token } + before do + download_artifact + end + it 'responds with forbidden' do expect(response).to have_gitlab_http_status(403) end @@ -1208,12 +1250,16 @@ describe API::Runner do context 'when job does not has artifacts' do it 'responds with not found' do + download_artifact + expect(response).to have_gitlab_http_status(404) end end def download_artifact(params = {}, request_headers = headers) params = params.merge(token: token) + job.reload + get api("/jobs/#{job.id}/artifacts"), params, request_headers end end diff --git a/spec/requests/api/v3/builds_spec.rb b/spec/requests/api/v3/builds_spec.rb index 79041c6a792..00f067889a0 100644 --- a/spec/requests/api/v3/builds_spec.rb +++ b/spec/requests/api/v3/builds_spec.rb @@ -216,6 +216,7 @@ describe API::V3::Builds do describe 'GET /projects/:id/builds/:build_id/artifacts' do before do + stub_artifacts_object_storage get v3_api("/projects/#{project.id}/builds/#{build.id}/artifacts", api_user) end @@ -230,13 +231,24 @@ describe API::V3::Builds do end it 'returns specific job artifacts' do - expect(response).to have_gitlab_http_status(200) + expect(response).to have_http_status(200) expect(response.headers).to include(download_headers) expect(response.body).to match_file(build.artifacts_file.file.file) end end end + context 'when artifacts are stored remotely' do + let(:build) { create(:ci_build, pipeline: pipeline) } + let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: build) } + + it 'returns location redirect' do + get v3_api("/projects/#{project.id}/builds/#{build.id}/artifacts", api_user) + + expect(response).to have_gitlab_http_status(302) + end + end + context 'unauthorized user' do let(:api_user) { nil } @@ -256,6 +268,7 @@ describe API::V3::Builds do let(:build) { create(:ci_build, :artifacts, pipeline: pipeline) } before do + stub_artifacts_object_storage build.success end @@ -318,9 +331,24 @@ describe API::V3::Builds do "attachment; filename=#{build.artifacts_file.filename}" } end - it { expect(response).to have_gitlab_http_status(200) } + it { expect(response).to have_http_status(200) } it { expect(response.headers).to include(download_headers) } end + + context 'when artifacts are stored remotely' do + let(:build) { create(:ci_build, pipeline: pipeline) } + let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: build) } + + before do + build.reload + + get v3_api("/projects/#{project.id}/builds/#{build.id}/artifacts", api_user) + end + + it 'returns location redirect' do + expect(response).to have_http_status(302) + end + end end context 'with regular branch' do diff --git a/spec/requests/lfs_http_spec.rb b/spec/requests/lfs_http_spec.rb index 971b45c411d..f7c04c19903 100644 --- a/spec/requests/lfs_http_spec.rb +++ b/spec/requests/lfs_http_spec.rb @@ -191,10 +191,12 @@ describe 'Git LFS API and storage' do describe 'when fetching lfs object' do let(:project) { create(:project) } let(:update_permissions) { } + let(:before_get) { } before do enable_lfs update_permissions + before_get get "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}", nil, headers end @@ -239,6 +241,21 @@ describe 'Git LFS API and storage' do end it_behaves_like 'responds with a file' + + context 'when LFS uses object storage' do + let(:before_get) do + stub_lfs_object_storage + lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) + end + + it 'responds with redirect' do + expect(response).to have_gitlab_http_status(302) + end + + it 'responds with the file location' do + expect(response.location).to include(lfs_object.reload.file.path) + end + end end end @@ -978,6 +995,32 @@ describe 'Git LFS API and storage' do end end + context 'and workhorse requests upload finalize for a new lfs object' do + before do + lfs_object.destroy + end + + context 'with object storage disabled' do + it "doesn't attempt to migrate file to object storage" do + expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async) + + put_finalize(with_tempfile: true) + end + end + + context 'with object storage enabled' do + before do + stub_lfs_object_storage(background_upload: true) + end + + it 'schedules migration of file to object storage' do + expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with('LfsObjectUploader', 'LfsObject', :file, kind_of(Numeric)) + + put_finalize(with_tempfile: true) + end + end + end + context 'invalid tempfiles' do it 'rejects slashes in the tempfile name (path traversal' do put_finalize('foo/bar') @@ -1177,7 +1220,9 @@ describe 'Git LFS API and storage' do put "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}/authorize", nil, authorize_headers end - def put_finalize(lfs_tmp = lfs_tmp_file) + def put_finalize(lfs_tmp = lfs_tmp_file, with_tempfile: false) + setup_tempfile(lfs_tmp) if with_tempfile + put "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}", nil, headers.merge('X-Gitlab-Lfs-Tmp' => lfs_tmp).compact end @@ -1185,6 +1230,13 @@ describe 'Git LFS API and storage' do def lfs_tmp_file "#{sample_oid}012345678" end + + def setup_tempfile(lfs_tmp) + upload_path = LfsObjectUploader.workhorse_upload_path + + FileUtils.mkdir_p(upload_path) + FileUtils.touch(File.join(upload_path, lfs_tmp)) + end end def enable_lfs diff --git a/spec/serializers/pipeline_serializer_spec.rb b/spec/serializers/pipeline_serializer_spec.rb index c38795ad1a1..f51c11b141f 100644 --- a/spec/serializers/pipeline_serializer_spec.rb +++ b/spec/serializers/pipeline_serializer_spec.rb @@ -117,6 +117,7 @@ describe PipelineSerializer do shared_examples 'no N+1 queries' do it 'verifies number of queries', :request_store do recorded = ActiveRecord::QueryRecorder.new { subject } + expect(recorded.count).to be_within(1).of(36) expect(recorded.cached_count).to eq(0) end diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index db9c216d3f4..b86a3d72bb4 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -28,7 +28,8 @@ describe Ci::RetryBuildService do %i[type lock_version target_url base_tags trace_sections commit_id deployments erased_by_id last_deployment project_id runner_id tag_taggings taggings tags trigger_request_id - user_id auto_canceled_by_id retried failure_reason].freeze + user_id auto_canceled_by_id retried failure_reason + artifacts_file_store artifacts_metadata_store].freeze shared_examples 'build duplication' do let(:another_pipeline) { create(:ci_empty_pipeline, project: project) } diff --git a/spec/services/issues/move_service_spec.rb b/spec/services/issues/move_service_spec.rb index c148a98569b..a9aee9e100f 100644 --- a/spec/services/issues/move_service_spec.rb +++ b/spec/services/issues/move_service_spec.rb @@ -6,7 +6,7 @@ describe Issues::MoveService do let(:title) { 'Some issue' } let(:description) { 'Some issue description' } let(:old_project) { create(:project) } - let(:new_project) { create(:project, group: create(:group)) } + let(:new_project) { create(:project) } let(:milestone1) { create(:milestone, project_id: old_project.id, title: 'v9.0') } let(:old_issue) do diff --git a/spec/support/http_io/http_io_helpers.rb b/spec/support/http_io/http_io_helpers.rb new file mode 100644 index 00000000000..31e07e720cd --- /dev/null +++ b/spec/support/http_io/http_io_helpers.rb @@ -0,0 +1,64 @@ +module HttpIOHelpers + def stub_remote_trace_206 + WebMock.stub_request(:get, remote_trace_url) + .to_return { |request| remote_trace_response(request, 206) } + end + + def stub_remote_trace_200 + WebMock.stub_request(:get, remote_trace_url) + .to_return { |request| remote_trace_response(request, 200) } + end + + def stub_remote_trace_500 + WebMock.stub_request(:get, remote_trace_url) + .to_return(status: [500, "Internal Server Error"]) + end + + def remote_trace_url + "http://trace.com/trace" + end + + def remote_trace_response(request, responce_status) + range = request.headers['Range'].match(/bytes=(\d+)-(\d+)/) + + { + status: responce_status, + headers: remote_trace_response_headers(responce_status, range[1].to_i, range[2].to_i), + body: range_trace_body(range[1].to_i, range[2].to_i) + } + end + + def remote_trace_response_headers(responce_status, from, to) + headers = { 'Content-Type' => 'text/plain' } + + if responce_status == 206 + headers.merge('Content-Range' => "bytes #{from}-#{to}/#{remote_trace_size}") + end + + headers + end + + def range_trace_body(from, to) + remote_trace_body[from..to] + end + + def remote_trace_body + @remote_trace_body ||= File.read(expand_fixture_path('trace/sample_trace')) + end + + def remote_trace_size + remote_trace_body.length + end + + def set_smaller_buffer_size_than(file_size) + blocks = (file_size / 128) + new_size = (blocks / 2) * 128 + stub_const("Gitlab::Ci::Trace::HttpIO::BUFFER_SIZE", new_size) + end + + def set_larger_buffer_size_than(file_size) + blocks = (file_size / 128) + new_size = (blocks * 2) * 128 + stub_const("Gitlab::Ci::Trace::HttpIO::BUFFER_SIZE", new_size) + end +end diff --git a/spec/support/shared_examples/uploaders/object_storage_shared_examples.rb b/spec/support/shared_examples/uploaders/object_storage_shared_examples.rb new file mode 100644 index 00000000000..cd9974cd6e2 --- /dev/null +++ b/spec/support/shared_examples/uploaders/object_storage_shared_examples.rb @@ -0,0 +1,126 @@ +shared_context 'with storage' do |store, **stub_params| + before do + subject.object_store = store + end +end + +shared_examples "migrates" do |to_store:, from_store: nil| + let(:to) { to_store } + let(:from) { from_store || subject.object_store } + + def migrate(to) + subject.migrate!(to) + end + + def checksum + Digest::SHA256.hexdigest(subject.read) + end + + before do + migrate(from) + end + + it 'returns corresponding file type' do + expect(subject).to be_an(CarrierWave::Uploader::Base) + expect(subject).to be_a(ObjectStorage::Concern) + + if from == described_class::Store::REMOTE + expect(subject.file).to be_a(CarrierWave::Storage::Fog::File) + elsif from == described_class::Store::LOCAL + expect(subject.file).to be_a(CarrierWave::SanitizedFile) + else + raise 'Unexpected file type' + end + end + + it 'does nothing when migrating to the current store' do + expect { migrate(from) }.not_to change { subject.object_store }.from(from) + end + + it 'migrate to the specified store' do + from_checksum = checksum + + expect { migrate(to) }.to change { subject.object_store }.from(from).to(to) + expect(checksum).to eq(from_checksum) + end + + it 'removes the original file after the migration' do + original_file = subject.file.path + migrate(to) + + expect(File.exist?(original_file)).to be_falsey + end + + it 'can access to the original file during migration' do + file = subject.file + + allow(subject).to receive(:delete_migrated_file) { } # Remove as a callback of :migrate + allow(subject).to receive(:record_upload) { } # Remove as a callback of :store (:record_upload) + + expect(file.exists?).to be_truthy + expect { migrate(to) }.not_to change { file.exists? } + end + + context 'when migrate! is not oqqupied by another process' do + it 'executes migrate!' do + expect(subject).to receive(:object_store=).at_least(1) + + migrate(to) + end + end + + context 'when migrate! is occupied by another process' do + let(:exclusive_lease_key) { "object_storage_migrate:#{subject.model.class}:#{subject.model.id}" } + + before do + @uuid = Gitlab::ExclusiveLease.new(exclusive_lease_key, timeout: 1.hour.to_i).try_obtain + end + + it 'does not execute migrate!' do + expect(subject).not_to receive(:unsafe_migrate!) + + expect { migrate(to) }.to raise_error('Already running') + end + + after do + Gitlab::ExclusiveLease.cancel(exclusive_lease_key, @uuid) + end + end + + context 'migration is unsuccessful' do + shared_examples "handles gracefully" do |error:| + it 'does not update the object_store' do + expect { migrate(to) }.to raise_error(error) + expect(subject.object_store).to eq(from) + end + + it 'does not delete the original file' do + expect { migrate(to) }.to raise_error(error) + expect(subject.exists?).to be_truthy + end + end + + context 'when the store is not supported' do + let(:to) { -1 } # not a valid store + + include_examples "handles gracefully", error: ObjectStorage::UnknownStoreError + end + + context 'upon a fog failure' do + before do + storage_class = subject.send(:storage_for, to).class + expect_any_instance_of(storage_class).to receive(:store!).and_raise("Store failure.") + end + + include_examples "handles gracefully", error: "Store failure." + end + + context 'upon a database failure' do + before do + expect(uploader).to receive(:persist_object_store!).and_raise("ActiveRecord failure.") + end + + include_examples "handles gracefully", error: "ActiveRecord failure." + end + end +end diff --git a/spec/support/stub_object_storage.rb b/spec/support/stub_object_storage.rb new file mode 100644 index 00000000000..1a0a2feb27d --- /dev/null +++ b/spec/support/stub_object_storage.rb @@ -0,0 +1,43 @@ +module StubConfiguration + def stub_object_storage_uploader( + config:, uploader:, remote_directory:, + enabled: true, + proxy_download: false, + background_upload: false) + Fog.mock! + + allow(config).to receive(:enabled) { enabled } + allow(config).to receive(:proxy_download) { proxy_download } + allow(config).to receive(:background_upload) { background_upload } + + return unless enabled + + ::Fog::Storage.new(uploader.object_store_credentials).tap do |connection| + begin + connection.directories.create(key: remote_directory) + rescue Excon::Error::Conflict + end + end + end + + def stub_artifacts_object_storage(**params) + stub_object_storage_uploader(config: Gitlab.config.artifacts.object_store, + uploader: JobArtifactUploader, + remote_directory: 'artifacts', + **params) + end + + def stub_lfs_object_storage(**params) + stub_object_storage_uploader(config: Gitlab.config.lfs.object_store, + uploader: LfsObjectUploader, + remote_directory: 'lfs-objects', + **params) + end + + def stub_uploads_object_storage(uploader = described_class, **params) + stub_object_storage_uploader(config: Gitlab.config.uploads.object_store, + uploader: uploader, + remote_directory: 'uploads', + **params) + end +end diff --git a/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb b/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb new file mode 100644 index 00000000000..8544fb62b5a --- /dev/null +++ b/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb @@ -0,0 +1,118 @@ +require 'rake_helper' + +describe 'gitlab:artifacts namespace rake task' do + before(:context) do + Rake.application.rake_require 'tasks/gitlab/artifacts/migrate' + end + + let(:object_storage_enabled) { false } + + before do + stub_artifacts_object_storage(enabled: object_storage_enabled) + end + + subject { run_rake_task('gitlab:artifacts:migrate') } + + context 'legacy artifacts' do + describe 'migrate' do + let!(:build) { create(:ci_build, :legacy_artifacts, artifacts_file_store: store, artifacts_metadata_store: store) } + + context 'when local storage is used' do + let(:store) { ObjectStorage::Store::LOCAL } + + context 'and job does not have file store defined' do + let(:object_storage_enabled) { true } + let(:store) { nil } + + it "migrates file to remote storage" do + subject + + expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE) + expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE) + end + end + + context 'and remote storage is defined' do + let(:object_storage_enabled) { true } + + it "migrates file to remote storage" do + subject + + expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE) + expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE) + end + end + + context 'and remote storage is not defined' do + it "fails to migrate to remote storage" do + subject + + expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::LOCAL) + expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::LOCAL) + end + end + end + + context 'when remote storage is used' do + let(:object_storage_enabled) { true } + + let(:store) { ObjectStorage::Store::REMOTE } + + it "file stays on remote storage" do + subject + + expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE) + expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE) + end + end + end + end + + context 'job artifacts' do + let!(:artifact) { create(:ci_job_artifact, :archive, file_store: store) } + + context 'when local storage is used' do + let(:store) { ObjectStorage::Store::LOCAL } + + context 'and job does not have file store defined' do + let(:object_storage_enabled) { true } + let(:store) { nil } + + it "migrates file to remote storage" do + subject + + expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + end + end + + context 'and remote storage is defined' do + let(:object_storage_enabled) { true } + + it "migrates file to remote storage" do + subject + + expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + end + end + + context 'and remote storage is not defined' do + it "fails to migrate to remote storage" do + subject + + expect(artifact.reload.file_store).to eq(ObjectStorage::Store::LOCAL) + end + end + end + + context 'when remote storage is used' do + let(:object_storage_enabled) { true } + let(:store) { ObjectStorage::Store::REMOTE } + + it "file stays on remote storage" do + subject + + expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + end + end + end +end diff --git a/spec/tasks/gitlab/lfs/migrate_rake_spec.rb b/spec/tasks/gitlab/lfs/migrate_rake_spec.rb new file mode 100644 index 00000000000..66d1a192a96 --- /dev/null +++ b/spec/tasks/gitlab/lfs/migrate_rake_spec.rb @@ -0,0 +1,37 @@ +require 'rake_helper' + +describe 'gitlab:lfs namespace rake task' do + before :all do + Rake.application.rake_require 'tasks/gitlab/lfs/migrate' + end + + describe 'migrate' do + let(:local) { ObjectStorage::Store::LOCAL } + let(:remote) { ObjectStorage::Store::REMOTE } + let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) } + + def lfs_migrate + run_rake_task('gitlab:lfs:migrate') + end + + context 'object storage disabled' do + before do + stub_lfs_object_storage(enabled: false) + end + + it "doesn't migrate files" do + expect { lfs_migrate }.not_to change { lfs_object.reload.file_store } + end + end + + context 'object storage enabled' do + before do + stub_lfs_object_storage + end + + it 'migrates local file to object storage' do + expect { lfs_migrate }.to change { lfs_object.reload.file_store }.from(local).to(remote) + end + end + end +end diff --git a/spec/tasks/gitlab/uploads/migrate_rake_spec.rb b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb new file mode 100644 index 00000000000..b778d26060d --- /dev/null +++ b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb @@ -0,0 +1,28 @@ +require 'rake_helper' + +describe 'gitlab:uploads:migrate rake tasks' do + let!(:projects) { create_list(:project, 10, :with_avatar) } + let(:model_class) { Project } + let(:uploader_class) { AvatarUploader } + let(:mounted_as) { :avatar } + let(:batch_size) { 3 } + + before do + stub_env('BATCH', batch_size.to_s) + stub_uploads_object_storage(uploader_class) + Rake.application.rake_require 'tasks/gitlab/uploads/migrate' + + allow(ObjectStorage::MigrateUploadsWorker).to receive(:perform_async) + end + + def run + args = [uploader_class.to_s, model_class.to_s, mounted_as].compact + run_rake_task("gitlab:uploads:migrate", *args) + end + + it 'enqueue jobs in batch' do + expect(ObjectStorage::MigrateUploadsWorker).to receive(:enqueue!).exactly(4).times + + run + end +end diff --git a/spec/uploaders/attachment_uploader_spec.rb b/spec/uploaders/attachment_uploader_spec.rb index 091ba824fc6..d302c14efb9 100644 --- a/spec/uploaders/attachment_uploader_spec.rb +++ b/spec/uploaders/attachment_uploader_spec.rb @@ -11,4 +11,26 @@ describe AttachmentUploader do store_dir: %r[uploads/-/system/note/attachment/], upload_path: %r[uploads/-/system/note/attachment/], absolute_path: %r[#{CarrierWave.root}/uploads/-/system/note/attachment/] + + context "object_store is REMOTE" do + before do + stub_uploads_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like 'builds correct paths', + store_dir: %r[note/attachment/], + upload_path: %r[note/attachment/] + end + + describe "#migrate!" do + before do + uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/doc_sample.txt'))) + stub_uploads_object_storage + end + + it_behaves_like "migrates", to_store: described_class::Store::REMOTE + it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL + end end diff --git a/spec/uploaders/avatar_uploader_spec.rb b/spec/uploaders/avatar_uploader_spec.rb index bf9028c9260..b0468bc35ff 100644 --- a/spec/uploaders/avatar_uploader_spec.rb +++ b/spec/uploaders/avatar_uploader_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe AvatarUploader do - let(:model) { create(:user, :with_avatar) } + let(:model) { build_stubbed(:user) } let(:uploader) { described_class.new(model, :avatar) } let(:upload) { create(:upload, model: model) } @@ -12,15 +12,28 @@ describe AvatarUploader do upload_path: %r[uploads/-/system/user/avatar/], absolute_path: %r[#{CarrierWave.root}/uploads/-/system/user/avatar/] - describe '#move_to_cache' do - it 'is false' do - expect(uploader.move_to_cache).to eq(false) + context "object_store is REMOTE" do + before do + stub_uploads_object_storage end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like 'builds correct paths', + store_dir: %r[user/avatar/], + upload_path: %r[user/avatar/] end - describe '#move_to_store' do - it 'is false' do - expect(uploader.move_to_store).to eq(false) + context "with a file" do + let(:project) { create(:project, :with_avatar) } + let(:uploader) { project.avatar } + let(:upload) { uploader.upload } + + before do + stub_uploads_object_storage end + + it_behaves_like "migrates", to_store: described_class::Store::REMOTE + it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL end end diff --git a/spec/uploaders/file_mover_spec.rb b/spec/uploaders/file_mover_spec.rb index bc024cd307c..68b7e24776d 100644 --- a/spec/uploaders/file_mover_spec.rb +++ b/spec/uploaders/file_mover_spec.rb @@ -36,6 +36,12 @@ describe FileMover do it 'creates a new update record' do expect { subject }.to change { Upload.count }.by(1) end + + it 'schedules a background migration' do + expect_any_instance_of(PersonalFileUploader).to receive(:schedule_background_upload).once + + subject + end end context 'when update_markdown fails' do diff --git a/spec/uploaders/file_uploader_spec.rb b/spec/uploaders/file_uploader_spec.rb index b42ce982b27..db2810bbe1d 100644 --- a/spec/uploaders/file_uploader_spec.rb +++ b/spec/uploaders/file_uploader_spec.rb @@ -11,32 +11,41 @@ describe FileUploader do shared_examples 'builds correct legacy storage paths' do include_examples 'builds correct paths', store_dir: %r{awesome/project/\h+}, + upload_path: %r{\h+/<filename>}, absolute_path: %r{#{described_class.root}/awesome/project/secret/foo.jpg} end - shared_examples 'uses hashed storage' do - context 'when rolled out attachments' do - let(:project) { build_stubbed(:project, namespace: group, name: 'project') } + context 'legacy storage' do + it_behaves_like 'builds correct legacy storage paths' - before do - allow(project).to receive(:disk_path).and_return('ca/fe/fe/ed') - end + context 'uses hashed storage' do + context 'when rolled out attachments' do + let(:project) { build_stubbed(:project, namespace: group, name: 'project') } - it_behaves_like 'builds correct paths', - store_dir: %r{ca/fe/fe/ed/\h+}, - absolute_path: %r{#{described_class.root}/ca/fe/fe/ed/secret/foo.jpg} - end + include_examples 'builds correct paths', + store_dir: %r{@hashed/\h{2}/\h{2}/\h+}, + upload_path: %r{\h+/<filename>} + end - context 'when only repositories are rolled out' do - let(:project) { build_stubbed(:project, namespace: group, name: 'project', storage_version: Project::HASHED_STORAGE_FEATURES[:repository]) } + context 'when only repositories are rolled out' do + let(:project) { build_stubbed(:project, namespace: group, name: 'project', storage_version: Project::HASHED_STORAGE_FEATURES[:repository]) } - it_behaves_like 'builds correct legacy storage paths' + it_behaves_like 'builds correct legacy storage paths' + end end end - context 'legacy storage' do - it_behaves_like 'builds correct legacy storage paths' - include_examples 'uses hashed storage' + context 'object store is remote' do + before do + stub_uploads_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + # always use hashed storage path for remote uploads + it_behaves_like 'builds correct paths', + store_dir: %r{@hashed/\h{2}/\h{2}/\h+}, + upload_path: %r{@hashed/\h{2}/\h{2}/\h+/\h+/<filename>} end describe 'initialize' do @@ -78,6 +87,16 @@ describe FileUploader do end end + describe "#migrate!" do + before do + uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/dk.png'))) + stub_uploads_object_storage + end + + it_behaves_like "migrates", to_store: described_class::Store::REMOTE + it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL + end + describe '#upload=' do let(:secret) { SecureRandom.hex } let(:upload) { create(:upload, :issuable_upload, secret: secret, filename: 'file.txt') } @@ -93,15 +112,5 @@ describe FileUploader do uploader.upload = upload end - - context 'uploader_context is empty' do - it 'fallbacks to regex based extraction' do - expect(upload).to receive(:uploader_context).and_return({}) - - uploader.upload = upload - expect(uploader.secret).to eq(secret) - expect(uploader.instance_variable_get(:@identifier)).to eq('file.txt') - end - end end end diff --git a/spec/uploaders/job_artifact_uploader_spec.rb b/spec/uploaders/job_artifact_uploader_spec.rb index 5612ec7e661..42036d67f3d 100644 --- a/spec/uploaders/job_artifact_uploader_spec.rb +++ b/spec/uploaders/job_artifact_uploader_spec.rb @@ -1,7 +1,8 @@ require 'spec_helper' describe JobArtifactUploader do - let(:job_artifact) { create(:ci_job_artifact) } + let(:store) { described_class::Store::LOCAL } + let(:job_artifact) { create(:ci_job_artifact, file_store: store) } let(:uploader) { described_class.new(job_artifact, :file) } subject { uploader } @@ -11,6 +12,17 @@ describe JobArtifactUploader do cache_dir: %r[artifacts/tmp/cache], work_dir: %r[artifacts/tmp/work] + context "object store is REMOTE" do + before do + stub_artifacts_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like "builds correct paths", + store_dir: %r[\h{2}/\h{2}/\h{64}/\d{4}_\d{1,2}_\d{1,2}/\d+/\d+\z] + end + describe '#open' do subject { uploader.open } @@ -36,6 +48,17 @@ describe JobArtifactUploader do end end end + + context 'when trace is stored in Object storage' do + before do + allow(uploader).to receive(:file_storage?) { false } + allow(uploader).to receive(:url) { 'http://object_storage.com/trace' } + end + + it 'returns http io stream' do + is_expected.to be_a(Gitlab::Ci::Trace::HttpIO) + end + end end context 'file is stored in valid local_path' do @@ -55,4 +78,14 @@ describe JobArtifactUploader do it { is_expected.to include("/#{job_artifact.job_id}/#{job_artifact.id}/") } it { is_expected.to end_with("ci_build_artifacts.zip") } end + + describe "#migrate!" do + before do + uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/trace/sample_trace'))) + stub_artifacts_object_storage + end + + it_behaves_like "migrates", to_store: described_class::Store::REMOTE + it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL + end end diff --git a/spec/uploaders/legacy_artifact_uploader_spec.rb b/spec/uploaders/legacy_artifact_uploader_spec.rb index 54c6a8b869b..eeb6fd90c9d 100644 --- a/spec/uploaders/legacy_artifact_uploader_spec.rb +++ b/spec/uploaders/legacy_artifact_uploader_spec.rb @@ -1,7 +1,8 @@ require 'rails_helper' describe LegacyArtifactUploader do - let(:job) { create(:ci_build) } + let(:store) { described_class::Store::LOCAL } + let(:job) { create(:ci_build, artifacts_file_store: store) } let(:uploader) { described_class.new(job, :legacy_artifacts_file) } let(:local_path) { described_class.root } @@ -20,6 +21,17 @@ describe LegacyArtifactUploader do cache_dir: %r[artifacts/tmp/cache], work_dir: %r[artifacts/tmp/work] + context 'object store is remote' do + before do + stub_artifacts_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like "builds correct paths", + store_dir: %r[\d{4}_\d{1,2}/\d+/\d+\z] + end + describe '#filename' do # we need to use uploader, as this makes to use mounter # which initialises uploader.file object diff --git a/spec/uploaders/lfs_object_uploader_spec.rb b/spec/uploaders/lfs_object_uploader_spec.rb index 6ebc885daa8..a2fb3886610 100644 --- a/spec/uploaders/lfs_object_uploader_spec.rb +++ b/spec/uploaders/lfs_object_uploader_spec.rb @@ -11,4 +11,62 @@ describe LfsObjectUploader do store_dir: %r[\h{2}/\h{2}], cache_dir: %r[/lfs-objects/tmp/cache], work_dir: %r[/lfs-objects/tmp/work] + + context "object store is REMOTE" do + before do + stub_lfs_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like "builds correct paths", + store_dir: %r[\h{2}/\h{2}] + end + + describe 'migration to object storage' do + context 'with object storage disabled' do + it "is skipped" do + expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async) + + lfs_object + end + end + + context 'with object storage enabled' do + before do + stub_lfs_object_storage(background_upload: true) + end + + it 'is scheduled to run after creation' do + expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with(described_class.name, 'LfsObject', :file, kind_of(Numeric)) + + lfs_object + end + end + end + + describe 'remote file' do + let(:remote) { described_class::Store::REMOTE } + let(:lfs_object) { create(:lfs_object, file_store: remote) } + + context 'with object storage enabled' do + before do + stub_lfs_object_storage + end + + it 'can store file remotely' do + allow(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async) + + store_file(lfs_object) + + expect(lfs_object.file_store).to eq remote + expect(lfs_object.file.path).not_to be_blank + end + end + end + + def store_file(lfs_object) + lfs_object.file = fixture_file_upload(Rails.root.join("spec/fixtures/dk.png"), "`/png") + lfs_object.save! + end end diff --git a/spec/uploaders/namespace_file_uploader_spec.rb b/spec/uploaders/namespace_file_uploader_spec.rb index 24a2fc0f72e..a8ba01d70b8 100644 --- a/spec/uploaders/namespace_file_uploader_spec.rb +++ b/spec/uploaders/namespace_file_uploader_spec.rb @@ -13,4 +13,26 @@ describe NamespaceFileUploader do store_dir: %r[uploads/-/system/namespace/\d+], upload_path: IDENTIFIER, absolute_path: %r[#{CarrierWave.root}/uploads/-/system/namespace/\d+/#{IDENTIFIER}] + + context "object_store is REMOTE" do + before do + stub_uploads_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like 'builds correct paths', + store_dir: %r[namespace/\d+/\h+], + upload_path: IDENTIFIER + end + + describe "#migrate!" do + before do + uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/doc_sample.txt'))) + stub_uploads_object_storage + end + + it_behaves_like "migrates", to_store: described_class::Store::REMOTE + it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL + end end diff --git a/spec/uploaders/object_storage_spec.rb b/spec/uploaders/object_storage_spec.rb new file mode 100644 index 00000000000..489b6707c6e --- /dev/null +++ b/spec/uploaders/object_storage_spec.rb @@ -0,0 +1,326 @@ +require 'rails_helper' +require 'carrierwave/storage/fog' + +class Implementation < GitlabUploader + include ObjectStorage::Concern + include ::RecordsUploads::Concern + prepend ::ObjectStorage::Extension::RecordsUploads + + storage_options Gitlab.config.uploads + + private + + # user/:id + def dynamic_segment + File.join(model.class.to_s.underscore, model.id.to_s) + end +end + +describe ObjectStorage do + let(:uploader_class) { Implementation } + let(:object) { build_stubbed(:user) } + let(:uploader) { uploader_class.new(object, :file) } + + before do + allow(uploader_class).to receive(:object_store_enabled?).and_return(true) + end + + describe '#object_store=' do + it "reload the local storage" do + uploader.object_store = described_class::Store::LOCAL + expect(uploader.file_storage?).to be_truthy + end + + it "reload the REMOTE storage" do + uploader.object_store = described_class::Store::REMOTE + expect(uploader.file_storage?).to be_falsey + end + end + + context 'object_store is Store::LOCAL' do + before do + uploader.object_store = described_class::Store::LOCAL + end + + describe '#store_dir' do + it 'is the composition of (base_dir, dynamic_segment)' do + expect(uploader.store_dir).to start_with("uploads/-/system/user/") + end + end + end + + context 'object_store is Store::REMOTE' do + before do + uploader.object_store = described_class::Store::REMOTE + end + + describe '#store_dir' do + it 'is the composition of (dynamic_segment)' do + expect(uploader.store_dir).to start_with("user/") + end + end + end + + describe '#object_store' do + it "delegates to <mount>_store on model" do + expect(object).to receive(:file_store) + + uploader.object_store + end + + context 'when store is null' do + before do + expect(object).to receive(:file_store).and_return(nil) + end + + it "returns Store::LOCAL" do + expect(uploader.object_store).to eq(described_class::Store::LOCAL) + end + end + + context 'when value is set' do + before do + expect(object).to receive(:file_store).and_return(described_class::Store::REMOTE) + end + + it "returns the given value" do + expect(uploader.object_store).to eq(described_class::Store::REMOTE) + end + end + end + + describe '#file_cache_storage?' do + context 'when file storage is used' do + before do + uploader_class.cache_storage(:file) + end + + it { expect(uploader).to be_file_cache_storage } + end + + context 'when is remote storage' do + before do + uploader_class.cache_storage(:fog) + end + + it { expect(uploader).not_to be_file_cache_storage } + end + end + + # this means the model shall include + # include RecordsUpload::Concern + # prepend ObjectStorage::Extension::RecordsUploads + # the object_store persistence is delegated to the `Upload` model. + # + context 'when persist_object_store? is false' do + let(:object) { create(:project, :with_avatar) } + let(:uploader) { object.avatar } + + it { expect(object).to be_a(Avatarable) } + it { expect(uploader.persist_object_store?).to be_falsey } + + describe 'delegates the object_store logic to the `Upload` model' do + it 'sets @upload to the found `upload`' do + expect(uploader.upload).to eq(uploader.upload) + end + + it 'sets @object_store to the `Upload` value' do + expect(uploader.object_store).to eq(uploader.upload.store) + end + end + + describe '#migrate!' do + let(:new_store) { ObjectStorage::Store::REMOTE } + + before do + stub_uploads_object_storage(uploader: AvatarUploader) + end + + subject { uploader.migrate!(new_store) } + + it 'persist @object_store to the recorded upload' do + subject + + expect(uploader.upload.store).to eq(new_store) + end + + describe 'fails' do + it 'is handled gracefully' do + store = uploader.object_store + expect_any_instance_of(Upload).to receive(:save!).and_raise("An error") + + expect { subject }.to raise_error("An error") + expect(uploader.exists?).to be_truthy + expect(uploader.upload.store).to eq(store) + end + end + end + end + + # this means the model holds an <mounted_as>_store attribute directly + # and do not delegate the object_store persistence to the `Upload` model. + # + context 'persist_object_store? is true' do + context 'when using JobArtifactsUploader' do + let(:store) { described_class::Store::LOCAL } + let(:object) { create(:ci_job_artifact, :archive, file_store: store) } + let(:uploader) { object.file } + + context 'checking described_class' do + it "uploader include described_class::Concern" do + expect(uploader).to be_a(described_class::Concern) + end + end + + describe '#use_file' do + context 'when file is stored locally' do + it "calls a regular path" do + expect { |b| uploader.use_file(&b) }.not_to yield_with_args(%r[tmp/cache]) + end + end + + context 'when file is stored remotely' do + let(:store) { described_class::Store::REMOTE } + + before do + stub_artifacts_object_storage + end + + it "calls a cache path" do + expect { |b| uploader.use_file(&b) }.to yield_with_args(%r[tmp/cache]) + end + end + end + + describe '#migrate!' do + subject { uploader.migrate!(new_store) } + + shared_examples "updates the underlying <mounted>_store" do + it do + subject + + expect(object.file_store).to eq(new_store) + end + end + + context 'when using the same storage' do + let(:new_store) { store } + + it "to not migrate the storage" do + subject + + expect(uploader).not_to receive(:store!) + expect(uploader.object_store).to eq(store) + end + end + + context 'when migrating to local storage' do + let(:store) { described_class::Store::REMOTE } + let(:new_store) { described_class::Store::LOCAL } + + before do + stub_artifacts_object_storage + end + + include_examples "updates the underlying <mounted>_store" + + it "local file does not exist" do + expect(File.exist?(uploader.path)).to eq(false) + end + + it "remote file exist" do + expect(uploader.file.exists?).to be_truthy + end + + it "does migrate the file" do + subject + + expect(uploader.object_store).to eq(new_store) + expect(File.exist?(uploader.path)).to eq(true) + end + end + + context 'when migrating to remote storage' do + let(:new_store) { described_class::Store::REMOTE } + let!(:current_path) { uploader.path } + + it "file does exist" do + expect(File.exist?(current_path)).to eq(true) + end + + context 'when storage is disabled' do + before do + stub_artifacts_object_storage(enabled: false) + end + + it "to raise an error" do + expect { subject }.to raise_error(/Object Storage is not enabled/) + end + end + + context 'when credentials are set' do + before do + stub_artifacts_object_storage + end + + include_examples "updates the underlying <mounted>_store" + + it "does migrate the file" do + subject + + expect(uploader.object_store).to eq(new_store) + end + + it "does delete original file" do + subject + + expect(File.exist?(current_path)).to eq(false) + end + + context 'when subject save fails' do + before do + expect(uploader).to receive(:persist_object_store!).and_raise(RuntimeError, "exception") + end + + it "original file is not removed" do + expect { subject }.to raise_error(/exception/) + + expect(File.exist?(current_path)).to eq(true) + end + end + end + end + end + end + end + + describe '#fog_directory' do + let(:remote_directory) { 'directory' } + + before do + uploader_class.storage_options double(object_store: double(remote_directory: remote_directory)) + end + + subject { uploader.fog_directory } + + it { is_expected.to eq(remote_directory) } + end + + describe '#fog_credentials' do + let(:connection) { Settingslogic.new("provider" => "AWS") } + + before do + uploader_class.storage_options double(object_store: double(connection: connection)) + end + + subject { uploader.fog_credentials } + + it { is_expected.to eq(provider: 'AWS') } + end + + describe '#fog_public' do + subject { uploader.fog_public } + + it { is_expected.to eq(false) } + end +end diff --git a/spec/uploaders/personal_file_uploader_spec.rb b/spec/uploaders/personal_file_uploader_spec.rb index ed1fba6edda..c70521d90dc 100644 --- a/spec/uploaders/personal_file_uploader_spec.rb +++ b/spec/uploaders/personal_file_uploader_spec.rb @@ -14,6 +14,18 @@ describe PersonalFileUploader do upload_path: IDENTIFIER, absolute_path: %r[#{CarrierWave.root}/uploads/-/system/personal_snippet/\d+/#{IDENTIFIER}] + context "object_store is REMOTE" do + before do + stub_uploads_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like 'builds correct paths', + store_dir: %r[\d+/\h+], + upload_path: IDENTIFIER + end + describe '#to_h' do before do subject.instance_variable_set(:@secret, 'secret') @@ -30,4 +42,14 @@ describe PersonalFileUploader do ) end end + + describe "#migrate!" do + before do + uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/doc_sample.txt'))) + stub_uploads_object_storage + end + + it_behaves_like "migrates", to_store: described_class::Store::REMOTE + it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL + end end diff --git a/spec/workers/object_storage_upload_worker_spec.rb b/spec/workers/object_storage_upload_worker_spec.rb new file mode 100644 index 00000000000..32ddcbe9757 --- /dev/null +++ b/spec/workers/object_storage_upload_worker_spec.rb @@ -0,0 +1,108 @@ +require 'spec_helper' + +describe ObjectStorageUploadWorker do + let(:local) { ObjectStorage::Store::LOCAL } + let(:remote) { ObjectStorage::Store::REMOTE } + + def perform + described_class.perform_async(uploader_class.name, subject_class, file_field, subject_id) + end + + context 'for LFS' do + let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) } + let(:uploader_class) { LfsObjectUploader } + let(:subject_class) { LfsObject } + let(:file_field) { :file } + let(:subject_id) { lfs_object.id } + + context 'when object storage is enabled' do + before do + stub_lfs_object_storage(background_upload: true) + end + + it 'uploads object to storage' do + expect { perform }.to change { lfs_object.reload.file_store }.from(local).to(remote) + end + + context 'when background upload is disabled' do + before do + allow(Gitlab.config.lfs.object_store).to receive(:background_upload) { false } + end + + it 'is skipped' do + expect { perform }.not_to change { lfs_object.reload.file_store } + end + end + end + + context 'when object storage is disabled' do + before do + stub_lfs_object_storage(enabled: false) + end + + it "doesn't migrate files" do + perform + + expect(lfs_object.reload.file_store).to eq(local) + end + end + end + + context 'for legacy artifacts' do + let(:build) { create(:ci_build, :legacy_artifacts) } + let(:uploader_class) { LegacyArtifactUploader } + let(:subject_class) { Ci::Build } + let(:file_field) { :artifacts_file } + let(:subject_id) { build.id } + + context 'when local storage is used' do + let(:store) { local } + + context 'and remote storage is defined' do + before do + stub_artifacts_object_storage(background_upload: true) + end + + it "migrates file to remote storage" do + perform + + expect(build.reload.artifacts_file_store).to eq(remote) + end + + context 'for artifacts_metadata' do + let(:file_field) { :artifacts_metadata } + + it 'migrates metadata to remote storage' do + perform + + expect(build.reload.artifacts_metadata_store).to eq(remote) + end + end + end + end + end + + context 'for job artifacts' do + let(:artifact) { create(:ci_job_artifact, :archive) } + let(:uploader_class) { JobArtifactUploader } + let(:subject_class) { Ci::JobArtifact } + let(:file_field) { :file } + let(:subject_id) { artifact.id } + + context 'when local storage is used' do + let(:store) { local } + + context 'and remote storage is defined' do + before do + stub_artifacts_object_storage(background_upload: true) + end + + it "migrates file to remote storage" do + perform + + expect(artifact.reload.file_store).to eq(remote) + end + end + end + end +end |