summaryrefslogtreecommitdiff
path: root/spec
diff options
context:
space:
mode:
Diffstat (limited to 'spec')
-rw-r--r--spec/controllers/concerns/send_file_upload_spec.rb89
-rw-r--r--spec/controllers/projects/artifacts_controller_spec.rb14
-rw-r--r--spec/controllers/projects/jobs_controller_spec.rb63
-rw-r--r--spec/controllers/projects/raw_controller_spec.rb56
-rw-r--r--spec/factories/appearances.rb17
-rw-r--r--spec/factories/ci/job_artifacts.rb4
-rw-r--r--spec/factories/lfs_objects.rb4
-rw-r--r--spec/factories/uploads.rb5
-rw-r--r--spec/initializers/fog_google_https_private_urls_spec.rb24
-rw-r--r--spec/lib/gitlab/ci/trace/http_io_spec.rb315
-rw-r--r--spec/lib/gitlab/import_export/safe_model_attributes.yml2
-rw-r--r--spec/lib/gitlab/verify/lfs_objects_spec.rb16
-rw-r--r--spec/lib/gitlab/verify/uploads_spec.rb16
-rw-r--r--spec/migrations/remove_empty_fork_networks_spec.rb4
-rw-r--r--spec/models/ci/build_spec.rb30
-rw-r--r--spec/models/ci/job_artifact_spec.rb44
-rw-r--r--spec/models/lfs_object_spec.rb85
-rw-r--r--spec/requests/api/jobs_spec.rb84
-rw-r--r--spec/requests/api/runner_spec.rb70
-rw-r--r--spec/requests/api/v3/builds_spec.rb32
-rw-r--r--spec/requests/lfs_http_spec.rb54
-rw-r--r--spec/serializers/pipeline_serializer_spec.rb1
-rw-r--r--spec/services/ci/retry_build_service_spec.rb3
-rw-r--r--spec/services/issues/move_service_spec.rb2
-rw-r--r--spec/support/http_io/http_io_helpers.rb64
-rw-r--r--spec/support/shared_examples/uploaders/object_storage_shared_examples.rb126
-rw-r--r--spec/support/stub_object_storage.rb43
-rw-r--r--spec/tasks/gitlab/artifacts/migrate_rake_spec.rb118
-rw-r--r--spec/tasks/gitlab/lfs/migrate_rake_spec.rb37
-rw-r--r--spec/tasks/gitlab/uploads/migrate_rake_spec.rb28
-rw-r--r--spec/uploaders/attachment_uploader_spec.rb22
-rw-r--r--spec/uploaders/avatar_uploader_spec.rb27
-rw-r--r--spec/uploaders/file_mover_spec.rb6
-rw-r--r--spec/uploaders/file_uploader_spec.rb61
-rw-r--r--spec/uploaders/job_artifact_uploader_spec.rb35
-rw-r--r--spec/uploaders/legacy_artifact_uploader_spec.rb14
-rw-r--r--spec/uploaders/lfs_object_uploader_spec.rb58
-rw-r--r--spec/uploaders/namespace_file_uploader_spec.rb22
-rw-r--r--spec/uploaders/object_storage_spec.rb326
-rw-r--r--spec/uploaders/personal_file_uploader_spec.rb22
-rw-r--r--spec/workers/object_storage_upload_worker_spec.rb108
41 files changed, 2065 insertions, 86 deletions
diff --git a/spec/controllers/concerns/send_file_upload_spec.rb b/spec/controllers/concerns/send_file_upload_spec.rb
new file mode 100644
index 00000000000..f4c99ea4064
--- /dev/null
+++ b/spec/controllers/concerns/send_file_upload_spec.rb
@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+describe SendFileUpload do
+ let(:uploader_class) do
+ Class.new(GitlabUploader) do
+ include ObjectStorage::Concern
+
+ storage_options Gitlab.config.uploads
+
+ private
+
+ # user/:id
+ def dynamic_segment
+ File.join(model.class.to_s.underscore, model.id.to_s)
+ end
+ end
+ end
+
+ let(:controller_class) do
+ Class.new do
+ include SendFileUpload
+ end
+ end
+
+ let(:object) { build_stubbed(:user) }
+ let(:uploader) { uploader_class.new(object, :file) }
+
+ describe '#send_upload' do
+ let(:controller) { controller_class.new }
+ let(:temp_file) { Tempfile.new('test') }
+
+ subject { controller.send_upload(uploader) }
+
+ before do
+ FileUtils.touch(temp_file)
+ end
+
+ after do
+ FileUtils.rm_f(temp_file)
+ end
+
+ context 'when local file is used' do
+ before do
+ uploader.store!(temp_file)
+ end
+
+ it 'sends a file' do
+ expect(controller).to receive(:send_file).with(uploader.path, anything)
+
+ subject
+ end
+ end
+
+ context 'when remote file is used' do
+ before do
+ stub_uploads_object_storage(uploader: uploader_class)
+ uploader.object_store = ObjectStorage::Store::REMOTE
+ uploader.store!(temp_file)
+ end
+
+ context 'and proxying is enabled' do
+ before do
+ allow(Gitlab.config.uploads.object_store).to receive(:proxy_download) { true }
+ end
+
+ it 'sends a file' do
+ headers = double
+ expect(headers).to receive(:store).with(Gitlab::Workhorse::SEND_DATA_HEADER, /^send-url:/)
+ expect(controller).to receive(:headers) { headers }
+ expect(controller).to receive(:head).with(:ok)
+
+ subject
+ end
+ end
+
+ context 'and proxying is disabled' do
+ before do
+ allow(Gitlab.config.uploads.object_store).to receive(:proxy_download) { false }
+ end
+
+ it 'sends a file' do
+ expect(controller).to receive(:redirect_to).with(/#{uploader.path}/)
+
+ subject
+ end
+ end
+ end
+ end
+end
diff --git a/spec/controllers/projects/artifacts_controller_spec.rb b/spec/controllers/projects/artifacts_controller_spec.rb
index 25a2e13fe1a..4ea6f869aa3 100644
--- a/spec/controllers/projects/artifacts_controller_spec.rb
+++ b/spec/controllers/projects/artifacts_controller_spec.rb
@@ -145,9 +145,23 @@ describe Projects::ArtifactsController do
context 'when using local file storage' do
it_behaves_like 'a valid file' do
let(:job) { create(:ci_build, :success, :artifacts, pipeline: pipeline) }
+ let(:store) { ObjectStorage::Store::LOCAL }
let(:archive_path) { JobArtifactUploader.root }
end
end
+
+ context 'when using remote file storage' do
+ before do
+ stub_artifacts_object_storage
+ end
+
+ it_behaves_like 'a valid file' do
+ let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) }
+ let!(:job) { create(:ci_build, :success, pipeline: pipeline) }
+ let(:store) { ObjectStorage::Store::REMOTE }
+ let(:archive_path) { 'https://' }
+ end
+ end
end
end
diff --git a/spec/controllers/projects/jobs_controller_spec.rb b/spec/controllers/projects/jobs_controller_spec.rb
index f3e303bb0fe..31046c202e6 100644
--- a/spec/controllers/projects/jobs_controller_spec.rb
+++ b/spec/controllers/projects/jobs_controller_spec.rb
@@ -1,7 +1,9 @@
+# coding: utf-8
require 'spec_helper'
describe Projects::JobsController do
include ApiHelpers
+ include HttpIOHelpers
let(:project) { create(:project, :public) }
let(:pipeline) { create(:ci_pipeline, project: project) }
@@ -203,6 +205,41 @@ describe Projects::JobsController do
end
end
+ context 'when trace artifact is in ObjectStorage' do
+ let!(:job) { create(:ci_build, :success, :trace_artifact, pipeline: pipeline) }
+
+ before do
+ allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false }
+ allow_any_instance_of(JobArtifactUploader).to receive(:url) { remote_trace_url }
+ allow_any_instance_of(JobArtifactUploader).to receive(:size) { remote_trace_size }
+ end
+
+ context 'when there are no network issues' do
+ before do
+ stub_remote_trace_206
+
+ get_trace
+ end
+
+ it 'returns a trace' do
+ expect(response).to have_gitlab_http_status(:ok)
+ expect(json_response['id']).to eq job.id
+ expect(json_response['status']).to eq job.status
+ expect(json_response['html']).to eq(job.trace.html)
+ end
+ end
+
+ context 'when there is a network issue' do
+ before do
+ stub_remote_trace_500
+ end
+
+ it 'returns a trace' do
+ expect { get_trace }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError)
+ end
+ end
+ end
+
def get_trace
get :trace, namespace_id: project.namespace,
project_id: project,
@@ -446,14 +483,18 @@ describe Projects::JobsController do
end
describe 'GET raw' do
- before do
- get_raw
+ subject do
+ post :raw, namespace_id: project.namespace,
+ project_id: project,
+ id: job.id
end
context 'when job has a trace artifact' do
let(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) }
it 'returns a trace' do
+ response = subject
+
expect(response).to have_gitlab_http_status(:ok)
expect(response.content_type).to eq 'text/plain; charset=utf-8'
expect(response.body).to eq job.job_artifacts_trace.open.read
@@ -464,6 +505,8 @@ describe Projects::JobsController do
let(:job) { create(:ci_build, :trace_live, pipeline: pipeline) }
it 'send a trace file' do
+ response = subject
+
expect(response).to have_gitlab_http_status(:ok)
expect(response.content_type).to eq 'text/plain; charset=utf-8'
expect(response.body).to eq 'BUILD TRACE'
@@ -474,14 +517,22 @@ describe Projects::JobsController do
let(:job) { create(:ci_build, pipeline: pipeline) }
it 'returns not_found' do
+ response = subject
+
expect(response).to have_gitlab_http_status(:not_found)
end
end
- def get_raw
- post :raw, namespace_id: project.namespace,
- project_id: project,
- id: job.id
+ context 'when the trace artifact is in ObjectStorage' do
+ let!(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) }
+
+ before do
+ allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false }
+ end
+
+ it 'redirect to the trace file url' do
+ expect(subject).to redirect_to(job.job_artifacts_trace.file.url)
+ end
end
end
end
diff --git a/spec/controllers/projects/raw_controller_spec.rb b/spec/controllers/projects/raw_controller_spec.rb
index b7df42168e0..08e2ccf893a 100644
--- a/spec/controllers/projects/raw_controller_spec.rb
+++ b/spec/controllers/projects/raw_controller_spec.rb
@@ -8,10 +8,7 @@ describe Projects::RawController do
let(:id) { 'master/README.md' }
it 'delivers ASCII file' do
- get(:show,
- namespace_id: public_project.namespace.to_param,
- project_id: public_project,
- id: id)
+ get_show(public_project, id)
expect(response).to have_gitlab_http_status(200)
expect(response.header['Content-Type']).to eq('text/plain; charset=utf-8')
@@ -25,10 +22,7 @@ describe Projects::RawController do
let(:id) { 'master/files/images/6049019_460s.jpg' }
it 'sets image content type header' do
- get(:show,
- namespace_id: public_project.namespace.to_param,
- project_id: public_project,
- id: id)
+ get_show(public_project, id)
expect(response).to have_gitlab_http_status(200)
expect(response.header['Content-Type']).to eq('image/jpeg')
@@ -54,21 +48,40 @@ describe Projects::RawController do
it 'serves the file' do
expect(controller).to receive(:send_file).with("#{LfsObjectUploader.root}/91/ef/f75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897", filename: 'lfs_object.iso', disposition: 'attachment')
- get(:show,
- namespace_id: public_project.namespace.to_param,
- project_id: public_project,
- id: id)
+ get_show(public_project, id)
expect(response).to have_gitlab_http_status(200)
end
+
+ context 'and lfs uses object storage' do
+ before do
+ lfs_object.file = fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png")
+ lfs_object.save!
+ stub_lfs_object_storage
+ lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
+ end
+
+ it 'responds with redirect to file' do
+ get_show(public_project, id)
+
+ expect(response).to have_gitlab_http_status(302)
+ expect(response.location).to include(lfs_object.reload.file.path)
+ end
+
+ it 'sets content disposition' do
+ get_show(public_project, id)
+
+ file_uri = URI.parse(response.location)
+ params = CGI.parse(file_uri.query)
+
+ expect(params["response-content-disposition"].first).to eq 'attachment;filename="lfs_object.iso"'
+ end
+ end
end
context 'when project does not have access' do
it 'does not serve the file' do
- get(:show,
- namespace_id: public_project.namespace.to_param,
- project_id: public_project,
- id: id)
+ get_show(public_project, id)
expect(response).to have_gitlab_http_status(404)
end
@@ -81,10 +94,7 @@ describe Projects::RawController do
end
it 'delivers ASCII file' do
- get(:show,
- namespace_id: public_project.namespace.to_param,
- project_id: public_project,
- id: id)
+ get_show(public_project, id)
expect(response).to have_gitlab_http_status(200)
expect(response.header['Content-Type']).to eq('text/plain; charset=utf-8')
@@ -95,4 +105,10 @@ describe Projects::RawController do
end
end
end
+
+ def get_show(project, id)
+ get(:show, namespace_id: project.namespace.to_param,
+ project_id: project,
+ id: id)
+ end
end
diff --git a/spec/factories/appearances.rb b/spec/factories/appearances.rb
index 5f9c57c0c8d..18c7453bd1b 100644
--- a/spec/factories/appearances.rb
+++ b/spec/factories/appearances.rb
@@ -2,8 +2,21 @@
FactoryBot.define do
factory :appearance do
- title "MepMep"
- description "This is my Community Edition instance"
+ title "GitLab Community Edition"
+ description "Open source software to collaborate on code"
new_project_guidelines "Custom project guidelines"
end
+
+ trait :with_logo do
+ logo { fixture_file_upload('spec/fixtures/dk.png') }
+ end
+
+ trait :with_header_logo do
+ header_logo { fixture_file_upload('spec/fixtures/dk.png') }
+ end
+
+ trait :with_logos do
+ with_logo
+ with_header_logo
+ end
end
diff --git a/spec/factories/ci/job_artifacts.rb b/spec/factories/ci/job_artifacts.rb
index 8544d54ccaa..3d3287d8168 100644
--- a/spec/factories/ci/job_artifacts.rb
+++ b/spec/factories/ci/job_artifacts.rb
@@ -5,6 +5,10 @@ FactoryBot.define do
job factory: :ci_build
file_type :archive
+ trait :remote_store do
+ file_store JobArtifactUploader::Store::REMOTE
+ end
+
after :build do |artifact|
artifact.project ||= artifact.job.project
end
diff --git a/spec/factories/lfs_objects.rb b/spec/factories/lfs_objects.rb
index caaed4d5246..eaf3a4ed497 100644
--- a/spec/factories/lfs_objects.rb
+++ b/spec/factories/lfs_objects.rb
@@ -15,4 +15,8 @@ FactoryBot.define do
trait :correct_oid do
oid 'b804383982bb89b00e828e3f44c038cc991d3d1768009fc39ba8e2c081b9fb75'
end
+
+ trait :object_storage do
+ file_store { LfsObjectUploader::Store::REMOTE }
+ end
end
diff --git a/spec/factories/uploads.rb b/spec/factories/uploads.rb
index ff3a2a76acc..b45f6f30e40 100644
--- a/spec/factories/uploads.rb
+++ b/spec/factories/uploads.rb
@@ -5,6 +5,7 @@ FactoryBot.define do
uploader "AvatarUploader"
mount_point :avatar
secret nil
+ store ObjectStorage::Store::LOCAL
# we should build a mount agnostic upload by default
transient do
@@ -27,6 +28,10 @@ FactoryBot.define do
secret SecureRandom.hex
end
+ trait :object_storage do
+ store ObjectStorage::Store::REMOTE
+ end
+
trait :namespace_upload do
model { build(:group) }
path { File.join(secret, filename) }
diff --git a/spec/initializers/fog_google_https_private_urls_spec.rb b/spec/initializers/fog_google_https_private_urls_spec.rb
new file mode 100644
index 00000000000..de3c157ab7b
--- /dev/null
+++ b/spec/initializers/fog_google_https_private_urls_spec.rb
@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+describe 'Fog::Storage::GoogleXML::File' do
+ let(:storage) do
+ Fog.mock!
+ Fog::Storage.new({
+ google_storage_access_key_id: "asdf",
+ google_storage_secret_access_key: "asdf",
+ provider: "Google"
+ })
+ end
+
+ let(:file) do
+ directory = storage.directories.create(key: 'data')
+ directory.files.create(
+ body: 'Hello World!',
+ key: 'hello_world.txt'
+ )
+ end
+
+ it 'delegates to #get_https_url' do
+ expect(file.url(Time.now)).to start_with("https://")
+ end
+end
diff --git a/spec/lib/gitlab/ci/trace/http_io_spec.rb b/spec/lib/gitlab/ci/trace/http_io_spec.rb
new file mode 100644
index 00000000000..5474e2f518c
--- /dev/null
+++ b/spec/lib/gitlab/ci/trace/http_io_spec.rb
@@ -0,0 +1,315 @@
+require 'spec_helper'
+
+describe Gitlab::Ci::Trace::HttpIO do
+ include HttpIOHelpers
+
+ let(:http_io) { described_class.new(url, size) }
+ let(:url) { remote_trace_url }
+ let(:size) { remote_trace_size }
+
+ describe '#close' do
+ subject { http_io.close }
+
+ it { is_expected.to be_nil }
+ end
+
+ describe '#binmode' do
+ subject { http_io.binmode }
+
+ it { is_expected.to be_nil }
+ end
+
+ describe '#binmode?' do
+ subject { http_io.binmode? }
+
+ it { is_expected.to be_truthy }
+ end
+
+ describe '#path' do
+ subject { http_io.path }
+
+ it { is_expected.to be_nil }
+ end
+
+ describe '#url' do
+ subject { http_io.url }
+
+ it { is_expected.to eq(url) }
+ end
+
+ describe '#seek' do
+ subject { http_io.seek(pos, where) }
+
+ context 'when moves pos to end of the file' do
+ let(:pos) { 0 }
+ let(:where) { IO::SEEK_END }
+
+ it { is_expected.to eq(size) }
+ end
+
+ context 'when moves pos to middle of the file' do
+ let(:pos) { size / 2 }
+ let(:where) { IO::SEEK_SET }
+
+ it { is_expected.to eq(size / 2) }
+ end
+
+ context 'when moves pos around' do
+ it 'matches the result' do
+ expect(http_io.seek(0)).to eq(0)
+ expect(http_io.seek(100, IO::SEEK_CUR)).to eq(100)
+ expect { http_io.seek(size + 1, IO::SEEK_CUR) }.to raise_error('new position is outside of file')
+ end
+ end
+ end
+
+ describe '#eof?' do
+ subject { http_io.eof? }
+
+ context 'when current pos is at end of the file' do
+ before do
+ http_io.seek(size, IO::SEEK_SET)
+ end
+
+ it { is_expected.to be_truthy }
+ end
+
+ context 'when current pos is not at end of the file' do
+ before do
+ http_io.seek(0, IO::SEEK_SET)
+ end
+
+ it { is_expected.to be_falsey }
+ end
+ end
+
+ describe '#each_line' do
+ subject { http_io.each_line }
+
+ let(:string_io) { StringIO.new(remote_trace_body) }
+
+ before do
+ stub_remote_trace_206
+ end
+
+ it 'yields lines' do
+ expect { |b| http_io.each_line(&b) }.to yield_successive_args(*string_io.each_line.to_a)
+ end
+
+ context 'when buckets on GCS' do
+ context 'when BUFFER_SIZE is larger than file size' do
+ before do
+ stub_remote_trace_200
+ set_larger_buffer_size_than(size)
+ end
+
+ it 'calls get_chunk only once' do
+ expect_any_instance_of(Net::HTTP).to receive(:request).once.and_call_original
+
+ http_io.each_line { |line| }
+ end
+ end
+ end
+ end
+
+ describe '#read' do
+ subject { http_io.read(length) }
+
+ context 'when there are no network issue' do
+ before do
+ stub_remote_trace_206
+ end
+
+ context 'when read whole size' do
+ let(:length) { nil }
+
+ context 'when BUFFER_SIZE is smaller than file size' do
+ before do
+ set_smaller_buffer_size_than(size)
+ end
+
+ it 'reads a trace' do
+ is_expected.to eq(remote_trace_body)
+ end
+ end
+
+ context 'when BUFFER_SIZE is larger than file size' do
+ before do
+ set_larger_buffer_size_than(size)
+ end
+
+ it 'reads a trace' do
+ is_expected.to eq(remote_trace_body)
+ end
+ end
+ end
+
+ context 'when read only first 100 bytes' do
+ let(:length) { 100 }
+
+ context 'when BUFFER_SIZE is smaller than file size' do
+ before do
+ set_smaller_buffer_size_than(size)
+ end
+
+ it 'reads a trace' do
+ is_expected.to eq(remote_trace_body[0, length])
+ end
+ end
+
+ context 'when BUFFER_SIZE is larger than file size' do
+ before do
+ set_larger_buffer_size_than(size)
+ end
+
+ it 'reads a trace' do
+ is_expected.to eq(remote_trace_body[0, length])
+ end
+ end
+ end
+
+ context 'when tries to read oversize' do
+ let(:length) { size + 1000 }
+
+ context 'when BUFFER_SIZE is smaller than file size' do
+ before do
+ set_smaller_buffer_size_than(size)
+ end
+
+ it 'reads a trace' do
+ is_expected.to eq(remote_trace_body)
+ end
+ end
+
+ context 'when BUFFER_SIZE is larger than file size' do
+ before do
+ set_larger_buffer_size_than(size)
+ end
+
+ it 'reads a trace' do
+ is_expected.to eq(remote_trace_body)
+ end
+ end
+ end
+
+ context 'when tries to read 0 bytes' do
+ let(:length) { 0 }
+
+ context 'when BUFFER_SIZE is smaller than file size' do
+ before do
+ set_smaller_buffer_size_than(size)
+ end
+
+ it 'reads a trace' do
+ is_expected.to be_empty
+ end
+ end
+
+ context 'when BUFFER_SIZE is larger than file size' do
+ before do
+ set_larger_buffer_size_than(size)
+ end
+
+ it 'reads a trace' do
+ is_expected.to be_empty
+ end
+ end
+ end
+ end
+
+ context 'when there is anetwork issue' do
+ let(:length) { nil }
+
+ before do
+ stub_remote_trace_500
+ end
+
+ it 'reads a trace' do
+ expect { subject }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError)
+ end
+ end
+ end
+
+ describe '#readline' do
+ subject { http_io.readline }
+
+ let(:string_io) { StringIO.new(remote_trace_body) }
+
+ before do
+ stub_remote_trace_206
+ end
+
+ shared_examples 'all line matching' do
+ it 'reads a line' do
+ (0...remote_trace_body.lines.count).each do
+ expect(http_io.readline).to eq(string_io.readline)
+ end
+ end
+ end
+
+ context 'when there is anetwork issue' do
+ let(:length) { nil }
+
+ before do
+ stub_remote_trace_500
+ end
+
+ it 'reads a trace' do
+ expect { subject }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError)
+ end
+ end
+
+ context 'when BUFFER_SIZE is smaller than file size' do
+ before do
+ set_smaller_buffer_size_than(size)
+ end
+
+ it_behaves_like 'all line matching'
+ end
+
+ context 'when BUFFER_SIZE is larger than file size' do
+ before do
+ set_larger_buffer_size_than(size)
+ end
+
+ it_behaves_like 'all line matching'
+ end
+
+ context 'when pos is at middle of the file' do
+ before do
+ set_smaller_buffer_size_than(size)
+
+ http_io.seek(size / 2)
+ string_io.seek(size / 2)
+ end
+
+ it 'reads from pos' do
+ expect(http_io.readline).to eq(string_io.readline)
+ end
+ end
+ end
+
+ describe '#write' do
+ subject { http_io.write(nil) }
+
+ it { expect { subject }.to raise_error(NotImplementedError) }
+ end
+
+ describe '#truncate' do
+ subject { http_io.truncate(nil) }
+
+ it { expect { subject }.to raise_error(NotImplementedError) }
+ end
+
+ describe '#flush' do
+ subject { http_io.flush }
+
+ it { expect { subject }.to raise_error(NotImplementedError) }
+ end
+
+ describe '#present?' do
+ subject { http_io.present? }
+
+ it { is_expected.to be_truthy }
+ end
+end
diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml
index 44e4c6ff94b..0716852f57f 100644
--- a/spec/lib/gitlab/import_export/safe_model_attributes.yml
+++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml
@@ -265,7 +265,9 @@ CommitStatus:
- target_url
- description
- artifacts_file
+- artifacts_file_store
- artifacts_metadata
+- artifacts_metadata_store
- erased_by_id
- erased_at
- artifacts_expire_at
diff --git a/spec/lib/gitlab/verify/lfs_objects_spec.rb b/spec/lib/gitlab/verify/lfs_objects_spec.rb
index 64f3a9660e0..0f890e2c7ce 100644
--- a/spec/lib/gitlab/verify/lfs_objects_spec.rb
+++ b/spec/lib/gitlab/verify/lfs_objects_spec.rb
@@ -31,5 +31,21 @@ describe Gitlab::Verify::LfsObjects do
expect(failures.keys).to contain_exactly(lfs_object)
expect(failure.to_s).to include('Checksum mismatch')
end
+
+ context 'with remote files' do
+ before do
+ stub_lfs_object_storage
+ end
+
+ it 'skips LFS objects in object storage' do
+ local_failure = create(:lfs_object)
+ create(:lfs_object, :object_storage)
+
+ failures = {}
+ described_class.new(batch_size: 10).run_batches { |_, failed| failures.merge!(failed) }
+
+ expect(failures.keys).to contain_exactly(local_failure)
+ end
+ end
end
end
diff --git a/spec/lib/gitlab/verify/uploads_spec.rb b/spec/lib/gitlab/verify/uploads_spec.rb
index 6146ce61226..85768308edc 100644
--- a/spec/lib/gitlab/verify/uploads_spec.rb
+++ b/spec/lib/gitlab/verify/uploads_spec.rb
@@ -40,5 +40,21 @@ describe Gitlab::Verify::Uploads do
expect(failures.keys).to contain_exactly(upload)
expect(failure.to_s).to include('Checksum missing')
end
+
+ context 'with remote files' do
+ before do
+ stub_uploads_object_storage(AvatarUploader)
+ end
+
+ it 'skips uploads in object storage' do
+ local_failure = create(:upload)
+ create(:upload, :object_storage)
+
+ failures = {}
+ described_class.new(batch_size: 10).run_batches { |_, failed| failures.merge!(failed) }
+
+ expect(failures.keys).to contain_exactly(local_failure)
+ end
+ end
end
end
diff --git a/spec/migrations/remove_empty_fork_networks_spec.rb b/spec/migrations/remove_empty_fork_networks_spec.rb
index 7f7ce91378b..f6d030ab25c 100644
--- a/spec/migrations/remove_empty_fork_networks_spec.rb
+++ b/spec/migrations/remove_empty_fork_networks_spec.rb
@@ -19,6 +19,10 @@ describe RemoveEmptyForkNetworks, :migration do
deleted_project.destroy!
end
+ after do
+ Upload.reset_column_information
+ end
+
it 'deletes only the fork network without members' do
expect(fork_networks.count).to eq(2)
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index 30a352fd090..7d935cf8d76 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -198,6 +198,16 @@ describe Ci::Build do
end
context 'when legacy artifacts are used' do
+ let(:build) { create(:ci_build, :legacy_artifacts) }
+
+ subject { build.artifacts? }
+
+ context 'is expired' do
+ let(:build) { create(:ci_build, :legacy_artifacts, :expired) }
+
+ it { is_expected.to be_falsy }
+ end
+
context 'artifacts archive does not exist' do
let(:build) { create(:ci_build) }
@@ -208,13 +218,25 @@ describe Ci::Build do
let(:build) { create(:ci_build, :legacy_artifacts) }
it { is_expected.to be_truthy }
+ end
+ end
+ end
- context 'is expired' do
- let(:build) { create(:ci_build, :legacy_artifacts, :expired) }
+ describe '#browsable_artifacts?' do
+ subject { build.browsable_artifacts? }
- it { is_expected.to be_falsy }
- end
+ context 'artifacts metadata does not exist' do
+ before do
+ build.update_attributes(legacy_artifacts_metadata: nil)
end
+
+ it { is_expected.to be_falsy }
+ end
+
+ context 'artifacts metadata does exists' do
+ let(:build) { create(:ci_build, :artifacts) }
+
+ it { is_expected.to be_truthy }
end
end
diff --git a/spec/models/ci/job_artifact_spec.rb b/spec/models/ci/job_artifact_spec.rb
index a2bd36537e6..1aa28434879 100644
--- a/spec/models/ci/job_artifact_spec.rb
+++ b/spec/models/ci/job_artifact_spec.rb
@@ -15,6 +15,50 @@ describe Ci::JobArtifact do
it { is_expected.to delegate_method(:open).to(:file) }
it { is_expected.to delegate_method(:exists?).to(:file) }
+ describe 'callbacks' do
+ subject { create(:ci_job_artifact, :archive) }
+
+ describe '#schedule_background_upload' do
+ context 'when object storage is disabled' do
+ before do
+ stub_artifacts_object_storage(enabled: false)
+ end
+
+ it 'does not schedule the migration' do
+ expect(ObjectStorageUploadWorker).not_to receive(:perform_async)
+
+ subject
+ end
+ end
+
+ context 'when object storage is enabled' do
+ context 'when background upload is enabled' do
+ before do
+ stub_artifacts_object_storage(background_upload: true)
+ end
+
+ it 'schedules the model for migration' do
+ expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with('JobArtifactUploader', described_class.name, :file, kind_of(Numeric))
+
+ subject
+ end
+ end
+
+ context 'when background upload is disabled' do
+ before do
+ stub_artifacts_object_storage(background_upload: false)
+ end
+
+ it 'schedules the model for migration' do
+ expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async)
+
+ subject
+ end
+ end
+ end
+ end
+ end
+
describe '#set_size' do
it 'sets the size' do
expect(artifact.size).to eq(106365)
diff --git a/spec/models/lfs_object_spec.rb b/spec/models/lfs_object_spec.rb
new file mode 100644
index 00000000000..a182116d637
--- /dev/null
+++ b/spec/models/lfs_object_spec.rb
@@ -0,0 +1,85 @@
+require 'spec_helper'
+
+describe LfsObject do
+ describe '#local_store?' do
+ it 'returns true when file_store is nil' do
+ subject.file_store = nil
+
+ expect(subject.local_store?).to eq true
+ end
+
+ it 'returns true when file_store is equal to LfsObjectUploader::Store::LOCAL' do
+ subject.file_store = LfsObjectUploader::Store::LOCAL
+
+ expect(subject.local_store?).to eq true
+ end
+
+ it 'returns false whe file_store is equal to LfsObjectUploader::Store::REMOTE' do
+ subject.file_store = LfsObjectUploader::Store::REMOTE
+
+ expect(subject.local_store?).to eq false
+ end
+ end
+
+ describe '#schedule_background_upload' do
+ before do
+ stub_lfs_setting(enabled: true)
+ end
+
+ subject { create(:lfs_object, :with_file) }
+
+ context 'when object storage is disabled' do
+ before do
+ stub_lfs_object_storage(enabled: false)
+ end
+
+ it 'does not schedule the migration' do
+ expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async)
+
+ subject
+ end
+ end
+
+ context 'when object storage is enabled' do
+ context 'when background upload is enabled' do
+ context 'when is licensed' do
+ before do
+ stub_lfs_object_storage(background_upload: true)
+ end
+
+ it 'schedules the model for migration' do
+ expect(ObjectStorage::BackgroundMoveWorker)
+ .to receive(:perform_async)
+ .with('LfsObjectUploader', described_class.name, :file, kind_of(Numeric))
+ .once
+
+ subject
+ end
+
+ it 'schedules the model for migration once' do
+ expect(ObjectStorage::BackgroundMoveWorker)
+ .to receive(:perform_async)
+ .with('LfsObjectUploader', described_class.name, :file, kind_of(Numeric))
+ .once
+
+ lfs_object = create(:lfs_object)
+ lfs_object.file = fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png")
+ lfs_object.save!
+ end
+ end
+ end
+
+ context 'when background upload is disabled' do
+ before do
+ stub_lfs_object_storage(background_upload: false)
+ end
+
+ it 'schedules the model for migration' do
+ expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async)
+
+ subject
+ end
+ end
+ end
+ end
+end
diff --git a/spec/requests/api/jobs_spec.rb b/spec/requests/api/jobs_spec.rb
index 6192bbd4abb..3ffdfdc0e9a 100644
--- a/spec/requests/api/jobs_spec.rb
+++ b/spec/requests/api/jobs_spec.rb
@@ -1,6 +1,8 @@
require 'spec_helper'
describe API::Jobs do
+ include HttpIOHelpers
+
set(:project) do
create(:project, :repository, public_builds: false)
end
@@ -112,6 +114,7 @@ describe API::Jobs do
let(:query) { Hash.new }
before do
+ job
get api("/projects/#{project.id}/pipelines/#{pipeline.id}/jobs", api_user), query
end
@@ -335,10 +338,55 @@ describe API::Jobs do
end
end
+ context 'when artifacts are stored remotely' do
+ let(:proxy_download) { false }
+
+ before do
+ stub_artifacts_object_storage(proxy_download: proxy_download)
+ end
+
+ let(:job) { create(:ci_build, pipeline: pipeline) }
+ let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) }
+
+ before do
+ job.reload
+
+ get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user)
+ end
+
+ context 'when proxy download is enabled' do
+ let(:proxy_download) { true }
+
+ it 'responds with the workhorse send-url' do
+ expect(response.headers[Gitlab::Workhorse::SEND_DATA_HEADER]).to start_with("send-url:")
+ end
+ end
+
+ context 'when proxy download is disabled' do
+ it 'returns location redirect' do
+ expect(response).to have_gitlab_http_status(302)
+ end
+ end
+
+ context 'authorized user' do
+ it 'returns the file remote URL' do
+ expect(response).to redirect_to(artifact.file.url)
+ end
+ end
+
+ context 'unauthorized user' do
+ let(:api_user) { nil }
+
+ it 'does not return specific job artifacts' do
+ expect(response).to have_gitlab_http_status(404)
+ end
+ end
+ end
+
it 'does not return job artifacts if not uploaded' do
get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user)
- expect(response).to have_gitlab_http_status(404)
+ expect(response).to have_gitlab_http_status(:not_found)
end
end
end
@@ -349,6 +397,7 @@ describe API::Jobs do
let(:job) { create(:ci_build, :artifacts, pipeline: pipeline, user: api_user) }
before do
+ stub_artifacts_object_storage
job.success
end
@@ -412,9 +461,24 @@ describe API::Jobs do
"attachment; filename=#{job.artifacts_file.filename}" }
end
- it { expect(response).to have_gitlab_http_status(200) }
+ it { expect(response).to have_http_status(:ok) }
it { expect(response.headers).to include(download_headers) }
end
+
+ context 'when artifacts are stored remotely' do
+ let(:job) { create(:ci_build, pipeline: pipeline, user: api_user) }
+ let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) }
+
+ before do
+ job.reload
+
+ get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user)
+ end
+
+ it 'returns location redirect' do
+ expect(response).to have_http_status(:found)
+ end
+ end
end
context 'with regular branch' do
@@ -451,6 +515,22 @@ describe API::Jobs do
end
context 'authorized user' do
+ context 'when trace is in ObjectStorage' do
+ let!(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) }
+
+ before do
+ stub_remote_trace_206
+ allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false }
+ allow_any_instance_of(JobArtifactUploader).to receive(:url) { remote_trace_url }
+ allow_any_instance_of(JobArtifactUploader).to receive(:size) { remote_trace_size }
+ end
+
+ it 'returns specific job trace' do
+ expect(response).to have_gitlab_http_status(200)
+ expect(response.body).to eq(job.trace.raw)
+ end
+ end
+
context 'when trace is artifact' do
let(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) }
diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb
index 95c23726a79..f3dd121faa9 100644
--- a/spec/requests/api/runner_spec.rb
+++ b/spec/requests/api/runner_spec.rb
@@ -200,7 +200,7 @@ describe API::Runner do
let(:project) { create(:project, shared_runners_enabled: false) }
let(:pipeline) { create(:ci_pipeline_without_jobs, project: project, ref: 'master') }
let(:runner) { create(:ci_runner) }
- let!(:job) do
+ let(:job) do
create(:ci_build, :artifacts, :extended_options,
pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0, commands: "ls\ndate")
end
@@ -215,6 +215,7 @@ describe API::Runner do
let(:user_agent) { 'gitlab-runner 9.0.0 (9-0-stable; go1.7.4; linux/amd64)' }
before do
+ job
stub_container_registry_config(enabled: false)
end
@@ -888,6 +889,7 @@ describe API::Runner do
let(:file_upload2) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/gif') }
before do
+ stub_artifacts_object_storage
job.run!
end
@@ -1179,27 +1181,67 @@ describe API::Runner do
describe 'GET /api/v4/jobs/:id/artifacts' do
let(:token) { job.token }
- before do
- download_artifact
- end
-
context 'when job has artifacts' do
- let(:job) { create(:ci_build, :artifacts) }
- let(:download_headers) do
- { 'Content-Transfer-Encoding' => 'binary',
- 'Content-Disposition' => 'attachment; filename=ci_build_artifacts.zip' }
+ let(:job) { create(:ci_build) }
+ let(:store) { JobArtifactUploader::Store::LOCAL }
+
+ before do
+ create(:ci_job_artifact, :archive, file_store: store, job: job)
end
context 'when using job token' do
- it 'download artifacts' do
- expect(response).to have_gitlab_http_status(200)
- expect(response.headers).to include download_headers
+ context 'when artifacts are stored locally' do
+ let(:download_headers) do
+ { 'Content-Transfer-Encoding' => 'binary',
+ 'Content-Disposition' => 'attachment; filename=ci_build_artifacts.zip' }
+ end
+
+ before do
+ download_artifact
+ end
+
+ it 'download artifacts' do
+ expect(response).to have_http_status(200)
+ expect(response.headers).to include download_headers
+ end
+ end
+
+ context 'when artifacts are stored remotely' do
+ let(:store) { JobArtifactUploader::Store::REMOTE }
+ let!(:job) { create(:ci_build) }
+
+ context 'when proxy download is being used' do
+ before do
+ download_artifact(direct_download: false)
+ end
+
+ it 'uses workhorse send-url' do
+ expect(response).to have_gitlab_http_status(200)
+ expect(response.headers).to include(
+ 'Gitlab-Workhorse-Send-Data' => /send-url:/)
+ end
+ end
+
+ context 'when direct download is being used' do
+ before do
+ download_artifact(direct_download: true)
+ end
+
+ it 'receive redirect for downloading artifacts' do
+ expect(response).to have_gitlab_http_status(302)
+ expect(response.headers).to include('Location')
+ end
+ end
end
end
context 'when using runnners token' do
let(:token) { job.project.runners_token }
+ before do
+ download_artifact
+ end
+
it 'responds with forbidden' do
expect(response).to have_gitlab_http_status(403)
end
@@ -1208,12 +1250,16 @@ describe API::Runner do
context 'when job does not has artifacts' do
it 'responds with not found' do
+ download_artifact
+
expect(response).to have_gitlab_http_status(404)
end
end
def download_artifact(params = {}, request_headers = headers)
params = params.merge(token: token)
+ job.reload
+
get api("/jobs/#{job.id}/artifacts"), params, request_headers
end
end
diff --git a/spec/requests/api/v3/builds_spec.rb b/spec/requests/api/v3/builds_spec.rb
index 79041c6a792..00f067889a0 100644
--- a/spec/requests/api/v3/builds_spec.rb
+++ b/spec/requests/api/v3/builds_spec.rb
@@ -216,6 +216,7 @@ describe API::V3::Builds do
describe 'GET /projects/:id/builds/:build_id/artifacts' do
before do
+ stub_artifacts_object_storage
get v3_api("/projects/#{project.id}/builds/#{build.id}/artifacts", api_user)
end
@@ -230,13 +231,24 @@ describe API::V3::Builds do
end
it 'returns specific job artifacts' do
- expect(response).to have_gitlab_http_status(200)
+ expect(response).to have_http_status(200)
expect(response.headers).to include(download_headers)
expect(response.body).to match_file(build.artifacts_file.file.file)
end
end
end
+ context 'when artifacts are stored remotely' do
+ let(:build) { create(:ci_build, pipeline: pipeline) }
+ let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: build) }
+
+ it 'returns location redirect' do
+ get v3_api("/projects/#{project.id}/builds/#{build.id}/artifacts", api_user)
+
+ expect(response).to have_gitlab_http_status(302)
+ end
+ end
+
context 'unauthorized user' do
let(:api_user) { nil }
@@ -256,6 +268,7 @@ describe API::V3::Builds do
let(:build) { create(:ci_build, :artifacts, pipeline: pipeline) }
before do
+ stub_artifacts_object_storage
build.success
end
@@ -318,9 +331,24 @@ describe API::V3::Builds do
"attachment; filename=#{build.artifacts_file.filename}" }
end
- it { expect(response).to have_gitlab_http_status(200) }
+ it { expect(response).to have_http_status(200) }
it { expect(response.headers).to include(download_headers) }
end
+
+ context 'when artifacts are stored remotely' do
+ let(:build) { create(:ci_build, pipeline: pipeline) }
+ let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: build) }
+
+ before do
+ build.reload
+
+ get v3_api("/projects/#{project.id}/builds/#{build.id}/artifacts", api_user)
+ end
+
+ it 'returns location redirect' do
+ expect(response).to have_http_status(302)
+ end
+ end
end
context 'with regular branch' do
diff --git a/spec/requests/lfs_http_spec.rb b/spec/requests/lfs_http_spec.rb
index 971b45c411d..f7c04c19903 100644
--- a/spec/requests/lfs_http_spec.rb
+++ b/spec/requests/lfs_http_spec.rb
@@ -191,10 +191,12 @@ describe 'Git LFS API and storage' do
describe 'when fetching lfs object' do
let(:project) { create(:project) }
let(:update_permissions) { }
+ let(:before_get) { }
before do
enable_lfs
update_permissions
+ before_get
get "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}", nil, headers
end
@@ -239,6 +241,21 @@ describe 'Git LFS API and storage' do
end
it_behaves_like 'responds with a file'
+
+ context 'when LFS uses object storage' do
+ let(:before_get) do
+ stub_lfs_object_storage
+ lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
+ end
+
+ it 'responds with redirect' do
+ expect(response).to have_gitlab_http_status(302)
+ end
+
+ it 'responds with the file location' do
+ expect(response.location).to include(lfs_object.reload.file.path)
+ end
+ end
end
end
@@ -978,6 +995,32 @@ describe 'Git LFS API and storage' do
end
end
+ context 'and workhorse requests upload finalize for a new lfs object' do
+ before do
+ lfs_object.destroy
+ end
+
+ context 'with object storage disabled' do
+ it "doesn't attempt to migrate file to object storage" do
+ expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async)
+
+ put_finalize(with_tempfile: true)
+ end
+ end
+
+ context 'with object storage enabled' do
+ before do
+ stub_lfs_object_storage(background_upload: true)
+ end
+
+ it 'schedules migration of file to object storage' do
+ expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with('LfsObjectUploader', 'LfsObject', :file, kind_of(Numeric))
+
+ put_finalize(with_tempfile: true)
+ end
+ end
+ end
+
context 'invalid tempfiles' do
it 'rejects slashes in the tempfile name (path traversal' do
put_finalize('foo/bar')
@@ -1177,7 +1220,9 @@ describe 'Git LFS API and storage' do
put "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}/authorize", nil, authorize_headers
end
- def put_finalize(lfs_tmp = lfs_tmp_file)
+ def put_finalize(lfs_tmp = lfs_tmp_file, with_tempfile: false)
+ setup_tempfile(lfs_tmp) if with_tempfile
+
put "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}", nil,
headers.merge('X-Gitlab-Lfs-Tmp' => lfs_tmp).compact
end
@@ -1185,6 +1230,13 @@ describe 'Git LFS API and storage' do
def lfs_tmp_file
"#{sample_oid}012345678"
end
+
+ def setup_tempfile(lfs_tmp)
+ upload_path = LfsObjectUploader.workhorse_upload_path
+
+ FileUtils.mkdir_p(upload_path)
+ FileUtils.touch(File.join(upload_path, lfs_tmp))
+ end
end
def enable_lfs
diff --git a/spec/serializers/pipeline_serializer_spec.rb b/spec/serializers/pipeline_serializer_spec.rb
index c38795ad1a1..f51c11b141f 100644
--- a/spec/serializers/pipeline_serializer_spec.rb
+++ b/spec/serializers/pipeline_serializer_spec.rb
@@ -117,6 +117,7 @@ describe PipelineSerializer do
shared_examples 'no N+1 queries' do
it 'verifies number of queries', :request_store do
recorded = ActiveRecord::QueryRecorder.new { subject }
+
expect(recorded.count).to be_within(1).of(36)
expect(recorded.cached_count).to eq(0)
end
diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb
index db9c216d3f4..b86a3d72bb4 100644
--- a/spec/services/ci/retry_build_service_spec.rb
+++ b/spec/services/ci/retry_build_service_spec.rb
@@ -28,7 +28,8 @@ describe Ci::RetryBuildService do
%i[type lock_version target_url base_tags trace_sections
commit_id deployments erased_by_id last_deployment project_id
runner_id tag_taggings taggings tags trigger_request_id
- user_id auto_canceled_by_id retried failure_reason].freeze
+ user_id auto_canceled_by_id retried failure_reason
+ artifacts_file_store artifacts_metadata_store].freeze
shared_examples 'build duplication' do
let(:another_pipeline) { create(:ci_empty_pipeline, project: project) }
diff --git a/spec/services/issues/move_service_spec.rb b/spec/services/issues/move_service_spec.rb
index c148a98569b..a9aee9e100f 100644
--- a/spec/services/issues/move_service_spec.rb
+++ b/spec/services/issues/move_service_spec.rb
@@ -6,7 +6,7 @@ describe Issues::MoveService do
let(:title) { 'Some issue' }
let(:description) { 'Some issue description' }
let(:old_project) { create(:project) }
- let(:new_project) { create(:project, group: create(:group)) }
+ let(:new_project) { create(:project) }
let(:milestone1) { create(:milestone, project_id: old_project.id, title: 'v9.0') }
let(:old_issue) do
diff --git a/spec/support/http_io/http_io_helpers.rb b/spec/support/http_io/http_io_helpers.rb
new file mode 100644
index 00000000000..31e07e720cd
--- /dev/null
+++ b/spec/support/http_io/http_io_helpers.rb
@@ -0,0 +1,64 @@
+module HttpIOHelpers
+ def stub_remote_trace_206
+ WebMock.stub_request(:get, remote_trace_url)
+ .to_return { |request| remote_trace_response(request, 206) }
+ end
+
+ def stub_remote_trace_200
+ WebMock.stub_request(:get, remote_trace_url)
+ .to_return { |request| remote_trace_response(request, 200) }
+ end
+
+ def stub_remote_trace_500
+ WebMock.stub_request(:get, remote_trace_url)
+ .to_return(status: [500, "Internal Server Error"])
+ end
+
+ def remote_trace_url
+ "http://trace.com/trace"
+ end
+
+ def remote_trace_response(request, responce_status)
+ range = request.headers['Range'].match(/bytes=(\d+)-(\d+)/)
+
+ {
+ status: responce_status,
+ headers: remote_trace_response_headers(responce_status, range[1].to_i, range[2].to_i),
+ body: range_trace_body(range[1].to_i, range[2].to_i)
+ }
+ end
+
+ def remote_trace_response_headers(responce_status, from, to)
+ headers = { 'Content-Type' => 'text/plain' }
+
+ if responce_status == 206
+ headers.merge('Content-Range' => "bytes #{from}-#{to}/#{remote_trace_size}")
+ end
+
+ headers
+ end
+
+ def range_trace_body(from, to)
+ remote_trace_body[from..to]
+ end
+
+ def remote_trace_body
+ @remote_trace_body ||= File.read(expand_fixture_path('trace/sample_trace'))
+ end
+
+ def remote_trace_size
+ remote_trace_body.length
+ end
+
+ def set_smaller_buffer_size_than(file_size)
+ blocks = (file_size / 128)
+ new_size = (blocks / 2) * 128
+ stub_const("Gitlab::Ci::Trace::HttpIO::BUFFER_SIZE", new_size)
+ end
+
+ def set_larger_buffer_size_than(file_size)
+ blocks = (file_size / 128)
+ new_size = (blocks * 2) * 128
+ stub_const("Gitlab::Ci::Trace::HttpIO::BUFFER_SIZE", new_size)
+ end
+end
diff --git a/spec/support/shared_examples/uploaders/object_storage_shared_examples.rb b/spec/support/shared_examples/uploaders/object_storage_shared_examples.rb
new file mode 100644
index 00000000000..cd9974cd6e2
--- /dev/null
+++ b/spec/support/shared_examples/uploaders/object_storage_shared_examples.rb
@@ -0,0 +1,126 @@
+shared_context 'with storage' do |store, **stub_params|
+ before do
+ subject.object_store = store
+ end
+end
+
+shared_examples "migrates" do |to_store:, from_store: nil|
+ let(:to) { to_store }
+ let(:from) { from_store || subject.object_store }
+
+ def migrate(to)
+ subject.migrate!(to)
+ end
+
+ def checksum
+ Digest::SHA256.hexdigest(subject.read)
+ end
+
+ before do
+ migrate(from)
+ end
+
+ it 'returns corresponding file type' do
+ expect(subject).to be_an(CarrierWave::Uploader::Base)
+ expect(subject).to be_a(ObjectStorage::Concern)
+
+ if from == described_class::Store::REMOTE
+ expect(subject.file).to be_a(CarrierWave::Storage::Fog::File)
+ elsif from == described_class::Store::LOCAL
+ expect(subject.file).to be_a(CarrierWave::SanitizedFile)
+ else
+ raise 'Unexpected file type'
+ end
+ end
+
+ it 'does nothing when migrating to the current store' do
+ expect { migrate(from) }.not_to change { subject.object_store }.from(from)
+ end
+
+ it 'migrate to the specified store' do
+ from_checksum = checksum
+
+ expect { migrate(to) }.to change { subject.object_store }.from(from).to(to)
+ expect(checksum).to eq(from_checksum)
+ end
+
+ it 'removes the original file after the migration' do
+ original_file = subject.file.path
+ migrate(to)
+
+ expect(File.exist?(original_file)).to be_falsey
+ end
+
+ it 'can access to the original file during migration' do
+ file = subject.file
+
+ allow(subject).to receive(:delete_migrated_file) { } # Remove as a callback of :migrate
+ allow(subject).to receive(:record_upload) { } # Remove as a callback of :store (:record_upload)
+
+ expect(file.exists?).to be_truthy
+ expect { migrate(to) }.not_to change { file.exists? }
+ end
+
+ context 'when migrate! is not oqqupied by another process' do
+ it 'executes migrate!' do
+ expect(subject).to receive(:object_store=).at_least(1)
+
+ migrate(to)
+ end
+ end
+
+ context 'when migrate! is occupied by another process' do
+ let(:exclusive_lease_key) { "object_storage_migrate:#{subject.model.class}:#{subject.model.id}" }
+
+ before do
+ @uuid = Gitlab::ExclusiveLease.new(exclusive_lease_key, timeout: 1.hour.to_i).try_obtain
+ end
+
+ it 'does not execute migrate!' do
+ expect(subject).not_to receive(:unsafe_migrate!)
+
+ expect { migrate(to) }.to raise_error('Already running')
+ end
+
+ after do
+ Gitlab::ExclusiveLease.cancel(exclusive_lease_key, @uuid)
+ end
+ end
+
+ context 'migration is unsuccessful' do
+ shared_examples "handles gracefully" do |error:|
+ it 'does not update the object_store' do
+ expect { migrate(to) }.to raise_error(error)
+ expect(subject.object_store).to eq(from)
+ end
+
+ it 'does not delete the original file' do
+ expect { migrate(to) }.to raise_error(error)
+ expect(subject.exists?).to be_truthy
+ end
+ end
+
+ context 'when the store is not supported' do
+ let(:to) { -1 } # not a valid store
+
+ include_examples "handles gracefully", error: ObjectStorage::UnknownStoreError
+ end
+
+ context 'upon a fog failure' do
+ before do
+ storage_class = subject.send(:storage_for, to).class
+ expect_any_instance_of(storage_class).to receive(:store!).and_raise("Store failure.")
+ end
+
+ include_examples "handles gracefully", error: "Store failure."
+ end
+
+ context 'upon a database failure' do
+ before do
+ expect(uploader).to receive(:persist_object_store!).and_raise("ActiveRecord failure.")
+ end
+
+ include_examples "handles gracefully", error: "ActiveRecord failure."
+ end
+ end
+end
diff --git a/spec/support/stub_object_storage.rb b/spec/support/stub_object_storage.rb
new file mode 100644
index 00000000000..1a0a2feb27d
--- /dev/null
+++ b/spec/support/stub_object_storage.rb
@@ -0,0 +1,43 @@
+module StubConfiguration
+ def stub_object_storage_uploader(
+ config:, uploader:, remote_directory:,
+ enabled: true,
+ proxy_download: false,
+ background_upload: false)
+ Fog.mock!
+
+ allow(config).to receive(:enabled) { enabled }
+ allow(config).to receive(:proxy_download) { proxy_download }
+ allow(config).to receive(:background_upload) { background_upload }
+
+ return unless enabled
+
+ ::Fog::Storage.new(uploader.object_store_credentials).tap do |connection|
+ begin
+ connection.directories.create(key: remote_directory)
+ rescue Excon::Error::Conflict
+ end
+ end
+ end
+
+ def stub_artifacts_object_storage(**params)
+ stub_object_storage_uploader(config: Gitlab.config.artifacts.object_store,
+ uploader: JobArtifactUploader,
+ remote_directory: 'artifacts',
+ **params)
+ end
+
+ def stub_lfs_object_storage(**params)
+ stub_object_storage_uploader(config: Gitlab.config.lfs.object_store,
+ uploader: LfsObjectUploader,
+ remote_directory: 'lfs-objects',
+ **params)
+ end
+
+ def stub_uploads_object_storage(uploader = described_class, **params)
+ stub_object_storage_uploader(config: Gitlab.config.uploads.object_store,
+ uploader: uploader,
+ remote_directory: 'uploads',
+ **params)
+ end
+end
diff --git a/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb b/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb
new file mode 100644
index 00000000000..8544fb62b5a
--- /dev/null
+++ b/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb
@@ -0,0 +1,118 @@
+require 'rake_helper'
+
+describe 'gitlab:artifacts namespace rake task' do
+ before(:context) do
+ Rake.application.rake_require 'tasks/gitlab/artifacts/migrate'
+ end
+
+ let(:object_storage_enabled) { false }
+
+ before do
+ stub_artifacts_object_storage(enabled: object_storage_enabled)
+ end
+
+ subject { run_rake_task('gitlab:artifacts:migrate') }
+
+ context 'legacy artifacts' do
+ describe 'migrate' do
+ let!(:build) { create(:ci_build, :legacy_artifacts, artifacts_file_store: store, artifacts_metadata_store: store) }
+
+ context 'when local storage is used' do
+ let(:store) { ObjectStorage::Store::LOCAL }
+
+ context 'and job does not have file store defined' do
+ let(:object_storage_enabled) { true }
+ let(:store) { nil }
+
+ it "migrates file to remote storage" do
+ subject
+
+ expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE)
+ expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE)
+ end
+ end
+
+ context 'and remote storage is defined' do
+ let(:object_storage_enabled) { true }
+
+ it "migrates file to remote storage" do
+ subject
+
+ expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE)
+ expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE)
+ end
+ end
+
+ context 'and remote storage is not defined' do
+ it "fails to migrate to remote storage" do
+ subject
+
+ expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::LOCAL)
+ expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::LOCAL)
+ end
+ end
+ end
+
+ context 'when remote storage is used' do
+ let(:object_storage_enabled) { true }
+
+ let(:store) { ObjectStorage::Store::REMOTE }
+
+ it "file stays on remote storage" do
+ subject
+
+ expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE)
+ expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE)
+ end
+ end
+ end
+ end
+
+ context 'job artifacts' do
+ let!(:artifact) { create(:ci_job_artifact, :archive, file_store: store) }
+
+ context 'when local storage is used' do
+ let(:store) { ObjectStorage::Store::LOCAL }
+
+ context 'and job does not have file store defined' do
+ let(:object_storage_enabled) { true }
+ let(:store) { nil }
+
+ it "migrates file to remote storage" do
+ subject
+
+ expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+ end
+ end
+
+ context 'and remote storage is defined' do
+ let(:object_storage_enabled) { true }
+
+ it "migrates file to remote storage" do
+ subject
+
+ expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+ end
+ end
+
+ context 'and remote storage is not defined' do
+ it "fails to migrate to remote storage" do
+ subject
+
+ expect(artifact.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+ end
+ end
+ end
+
+ context 'when remote storage is used' do
+ let(:object_storage_enabled) { true }
+ let(:store) { ObjectStorage::Store::REMOTE }
+
+ it "file stays on remote storage" do
+ subject
+
+ expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+ end
+ end
+ end
+end
diff --git a/spec/tasks/gitlab/lfs/migrate_rake_spec.rb b/spec/tasks/gitlab/lfs/migrate_rake_spec.rb
new file mode 100644
index 00000000000..66d1a192a96
--- /dev/null
+++ b/spec/tasks/gitlab/lfs/migrate_rake_spec.rb
@@ -0,0 +1,37 @@
+require 'rake_helper'
+
+describe 'gitlab:lfs namespace rake task' do
+ before :all do
+ Rake.application.rake_require 'tasks/gitlab/lfs/migrate'
+ end
+
+ describe 'migrate' do
+ let(:local) { ObjectStorage::Store::LOCAL }
+ let(:remote) { ObjectStorage::Store::REMOTE }
+ let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) }
+
+ def lfs_migrate
+ run_rake_task('gitlab:lfs:migrate')
+ end
+
+ context 'object storage disabled' do
+ before do
+ stub_lfs_object_storage(enabled: false)
+ end
+
+ it "doesn't migrate files" do
+ expect { lfs_migrate }.not_to change { lfs_object.reload.file_store }
+ end
+ end
+
+ context 'object storage enabled' do
+ before do
+ stub_lfs_object_storage
+ end
+
+ it 'migrates local file to object storage' do
+ expect { lfs_migrate }.to change { lfs_object.reload.file_store }.from(local).to(remote)
+ end
+ end
+ end
+end
diff --git a/spec/tasks/gitlab/uploads/migrate_rake_spec.rb b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb
new file mode 100644
index 00000000000..b778d26060d
--- /dev/null
+++ b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb
@@ -0,0 +1,28 @@
+require 'rake_helper'
+
+describe 'gitlab:uploads:migrate rake tasks' do
+ let!(:projects) { create_list(:project, 10, :with_avatar) }
+ let(:model_class) { Project }
+ let(:uploader_class) { AvatarUploader }
+ let(:mounted_as) { :avatar }
+ let(:batch_size) { 3 }
+
+ before do
+ stub_env('BATCH', batch_size.to_s)
+ stub_uploads_object_storage(uploader_class)
+ Rake.application.rake_require 'tasks/gitlab/uploads/migrate'
+
+ allow(ObjectStorage::MigrateUploadsWorker).to receive(:perform_async)
+ end
+
+ def run
+ args = [uploader_class.to_s, model_class.to_s, mounted_as].compact
+ run_rake_task("gitlab:uploads:migrate", *args)
+ end
+
+ it 'enqueue jobs in batch' do
+ expect(ObjectStorage::MigrateUploadsWorker).to receive(:enqueue!).exactly(4).times
+
+ run
+ end
+end
diff --git a/spec/uploaders/attachment_uploader_spec.rb b/spec/uploaders/attachment_uploader_spec.rb
index 091ba824fc6..d302c14efb9 100644
--- a/spec/uploaders/attachment_uploader_spec.rb
+++ b/spec/uploaders/attachment_uploader_spec.rb
@@ -11,4 +11,26 @@ describe AttachmentUploader do
store_dir: %r[uploads/-/system/note/attachment/],
upload_path: %r[uploads/-/system/note/attachment/],
absolute_path: %r[#{CarrierWave.root}/uploads/-/system/note/attachment/]
+
+ context "object_store is REMOTE" do
+ before do
+ stub_uploads_object_storage
+ end
+
+ include_context 'with storage', described_class::Store::REMOTE
+
+ it_behaves_like 'builds correct paths',
+ store_dir: %r[note/attachment/],
+ upload_path: %r[note/attachment/]
+ end
+
+ describe "#migrate!" do
+ before do
+ uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/doc_sample.txt')))
+ stub_uploads_object_storage
+ end
+
+ it_behaves_like "migrates", to_store: described_class::Store::REMOTE
+ it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL
+ end
end
diff --git a/spec/uploaders/avatar_uploader_spec.rb b/spec/uploaders/avatar_uploader_spec.rb
index bf9028c9260..b0468bc35ff 100644
--- a/spec/uploaders/avatar_uploader_spec.rb
+++ b/spec/uploaders/avatar_uploader_spec.rb
@@ -1,7 +1,7 @@
require 'spec_helper'
describe AvatarUploader do
- let(:model) { create(:user, :with_avatar) }
+ let(:model) { build_stubbed(:user) }
let(:uploader) { described_class.new(model, :avatar) }
let(:upload) { create(:upload, model: model) }
@@ -12,15 +12,28 @@ describe AvatarUploader do
upload_path: %r[uploads/-/system/user/avatar/],
absolute_path: %r[#{CarrierWave.root}/uploads/-/system/user/avatar/]
- describe '#move_to_cache' do
- it 'is false' do
- expect(uploader.move_to_cache).to eq(false)
+ context "object_store is REMOTE" do
+ before do
+ stub_uploads_object_storage
end
+
+ include_context 'with storage', described_class::Store::REMOTE
+
+ it_behaves_like 'builds correct paths',
+ store_dir: %r[user/avatar/],
+ upload_path: %r[user/avatar/]
end
- describe '#move_to_store' do
- it 'is false' do
- expect(uploader.move_to_store).to eq(false)
+ context "with a file" do
+ let(:project) { create(:project, :with_avatar) }
+ let(:uploader) { project.avatar }
+ let(:upload) { uploader.upload }
+
+ before do
+ stub_uploads_object_storage
end
+
+ it_behaves_like "migrates", to_store: described_class::Store::REMOTE
+ it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL
end
end
diff --git a/spec/uploaders/file_mover_spec.rb b/spec/uploaders/file_mover_spec.rb
index bc024cd307c..68b7e24776d 100644
--- a/spec/uploaders/file_mover_spec.rb
+++ b/spec/uploaders/file_mover_spec.rb
@@ -36,6 +36,12 @@ describe FileMover do
it 'creates a new update record' do
expect { subject }.to change { Upload.count }.by(1)
end
+
+ it 'schedules a background migration' do
+ expect_any_instance_of(PersonalFileUploader).to receive(:schedule_background_upload).once
+
+ subject
+ end
end
context 'when update_markdown fails' do
diff --git a/spec/uploaders/file_uploader_spec.rb b/spec/uploaders/file_uploader_spec.rb
index b42ce982b27..db2810bbe1d 100644
--- a/spec/uploaders/file_uploader_spec.rb
+++ b/spec/uploaders/file_uploader_spec.rb
@@ -11,32 +11,41 @@ describe FileUploader do
shared_examples 'builds correct legacy storage paths' do
include_examples 'builds correct paths',
store_dir: %r{awesome/project/\h+},
+ upload_path: %r{\h+/<filename>},
absolute_path: %r{#{described_class.root}/awesome/project/secret/foo.jpg}
end
- shared_examples 'uses hashed storage' do
- context 'when rolled out attachments' do
- let(:project) { build_stubbed(:project, namespace: group, name: 'project') }
+ context 'legacy storage' do
+ it_behaves_like 'builds correct legacy storage paths'
- before do
- allow(project).to receive(:disk_path).and_return('ca/fe/fe/ed')
- end
+ context 'uses hashed storage' do
+ context 'when rolled out attachments' do
+ let(:project) { build_stubbed(:project, namespace: group, name: 'project') }
- it_behaves_like 'builds correct paths',
- store_dir: %r{ca/fe/fe/ed/\h+},
- absolute_path: %r{#{described_class.root}/ca/fe/fe/ed/secret/foo.jpg}
- end
+ include_examples 'builds correct paths',
+ store_dir: %r{@hashed/\h{2}/\h{2}/\h+},
+ upload_path: %r{\h+/<filename>}
+ end
- context 'when only repositories are rolled out' do
- let(:project) { build_stubbed(:project, namespace: group, name: 'project', storage_version: Project::HASHED_STORAGE_FEATURES[:repository]) }
+ context 'when only repositories are rolled out' do
+ let(:project) { build_stubbed(:project, namespace: group, name: 'project', storage_version: Project::HASHED_STORAGE_FEATURES[:repository]) }
- it_behaves_like 'builds correct legacy storage paths'
+ it_behaves_like 'builds correct legacy storage paths'
+ end
end
end
- context 'legacy storage' do
- it_behaves_like 'builds correct legacy storage paths'
- include_examples 'uses hashed storage'
+ context 'object store is remote' do
+ before do
+ stub_uploads_object_storage
+ end
+
+ include_context 'with storage', described_class::Store::REMOTE
+
+ # always use hashed storage path for remote uploads
+ it_behaves_like 'builds correct paths',
+ store_dir: %r{@hashed/\h{2}/\h{2}/\h+},
+ upload_path: %r{@hashed/\h{2}/\h{2}/\h+/\h+/<filename>}
end
describe 'initialize' do
@@ -78,6 +87,16 @@ describe FileUploader do
end
end
+ describe "#migrate!" do
+ before do
+ uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/dk.png')))
+ stub_uploads_object_storage
+ end
+
+ it_behaves_like "migrates", to_store: described_class::Store::REMOTE
+ it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL
+ end
+
describe '#upload=' do
let(:secret) { SecureRandom.hex }
let(:upload) { create(:upload, :issuable_upload, secret: secret, filename: 'file.txt') }
@@ -93,15 +112,5 @@ describe FileUploader do
uploader.upload = upload
end
-
- context 'uploader_context is empty' do
- it 'fallbacks to regex based extraction' do
- expect(upload).to receive(:uploader_context).and_return({})
-
- uploader.upload = upload
- expect(uploader.secret).to eq(secret)
- expect(uploader.instance_variable_get(:@identifier)).to eq('file.txt')
- end
- end
end
end
diff --git a/spec/uploaders/job_artifact_uploader_spec.rb b/spec/uploaders/job_artifact_uploader_spec.rb
index 5612ec7e661..42036d67f3d 100644
--- a/spec/uploaders/job_artifact_uploader_spec.rb
+++ b/spec/uploaders/job_artifact_uploader_spec.rb
@@ -1,7 +1,8 @@
require 'spec_helper'
describe JobArtifactUploader do
- let(:job_artifact) { create(:ci_job_artifact) }
+ let(:store) { described_class::Store::LOCAL }
+ let(:job_artifact) { create(:ci_job_artifact, file_store: store) }
let(:uploader) { described_class.new(job_artifact, :file) }
subject { uploader }
@@ -11,6 +12,17 @@ describe JobArtifactUploader do
cache_dir: %r[artifacts/tmp/cache],
work_dir: %r[artifacts/tmp/work]
+ context "object store is REMOTE" do
+ before do
+ stub_artifacts_object_storage
+ end
+
+ include_context 'with storage', described_class::Store::REMOTE
+
+ it_behaves_like "builds correct paths",
+ store_dir: %r[\h{2}/\h{2}/\h{64}/\d{4}_\d{1,2}_\d{1,2}/\d+/\d+\z]
+ end
+
describe '#open' do
subject { uploader.open }
@@ -36,6 +48,17 @@ describe JobArtifactUploader do
end
end
end
+
+ context 'when trace is stored in Object storage' do
+ before do
+ allow(uploader).to receive(:file_storage?) { false }
+ allow(uploader).to receive(:url) { 'http://object_storage.com/trace' }
+ end
+
+ it 'returns http io stream' do
+ is_expected.to be_a(Gitlab::Ci::Trace::HttpIO)
+ end
+ end
end
context 'file is stored in valid local_path' do
@@ -55,4 +78,14 @@ describe JobArtifactUploader do
it { is_expected.to include("/#{job_artifact.job_id}/#{job_artifact.id}/") }
it { is_expected.to end_with("ci_build_artifacts.zip") }
end
+
+ describe "#migrate!" do
+ before do
+ uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/trace/sample_trace')))
+ stub_artifacts_object_storage
+ end
+
+ it_behaves_like "migrates", to_store: described_class::Store::REMOTE
+ it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL
+ end
end
diff --git a/spec/uploaders/legacy_artifact_uploader_spec.rb b/spec/uploaders/legacy_artifact_uploader_spec.rb
index 54c6a8b869b..eeb6fd90c9d 100644
--- a/spec/uploaders/legacy_artifact_uploader_spec.rb
+++ b/spec/uploaders/legacy_artifact_uploader_spec.rb
@@ -1,7 +1,8 @@
require 'rails_helper'
describe LegacyArtifactUploader do
- let(:job) { create(:ci_build) }
+ let(:store) { described_class::Store::LOCAL }
+ let(:job) { create(:ci_build, artifacts_file_store: store) }
let(:uploader) { described_class.new(job, :legacy_artifacts_file) }
let(:local_path) { described_class.root }
@@ -20,6 +21,17 @@ describe LegacyArtifactUploader do
cache_dir: %r[artifacts/tmp/cache],
work_dir: %r[artifacts/tmp/work]
+ context 'object store is remote' do
+ before do
+ stub_artifacts_object_storage
+ end
+
+ include_context 'with storage', described_class::Store::REMOTE
+
+ it_behaves_like "builds correct paths",
+ store_dir: %r[\d{4}_\d{1,2}/\d+/\d+\z]
+ end
+
describe '#filename' do
# we need to use uploader, as this makes to use mounter
# which initialises uploader.file object
diff --git a/spec/uploaders/lfs_object_uploader_spec.rb b/spec/uploaders/lfs_object_uploader_spec.rb
index 6ebc885daa8..a2fb3886610 100644
--- a/spec/uploaders/lfs_object_uploader_spec.rb
+++ b/spec/uploaders/lfs_object_uploader_spec.rb
@@ -11,4 +11,62 @@ describe LfsObjectUploader do
store_dir: %r[\h{2}/\h{2}],
cache_dir: %r[/lfs-objects/tmp/cache],
work_dir: %r[/lfs-objects/tmp/work]
+
+ context "object store is REMOTE" do
+ before do
+ stub_lfs_object_storage
+ end
+
+ include_context 'with storage', described_class::Store::REMOTE
+
+ it_behaves_like "builds correct paths",
+ store_dir: %r[\h{2}/\h{2}]
+ end
+
+ describe 'migration to object storage' do
+ context 'with object storage disabled' do
+ it "is skipped" do
+ expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async)
+
+ lfs_object
+ end
+ end
+
+ context 'with object storage enabled' do
+ before do
+ stub_lfs_object_storage(background_upload: true)
+ end
+
+ it 'is scheduled to run after creation' do
+ expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with(described_class.name, 'LfsObject', :file, kind_of(Numeric))
+
+ lfs_object
+ end
+ end
+ end
+
+ describe 'remote file' do
+ let(:remote) { described_class::Store::REMOTE }
+ let(:lfs_object) { create(:lfs_object, file_store: remote) }
+
+ context 'with object storage enabled' do
+ before do
+ stub_lfs_object_storage
+ end
+
+ it 'can store file remotely' do
+ allow(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async)
+
+ store_file(lfs_object)
+
+ expect(lfs_object.file_store).to eq remote
+ expect(lfs_object.file.path).not_to be_blank
+ end
+ end
+ end
+
+ def store_file(lfs_object)
+ lfs_object.file = fixture_file_upload(Rails.root.join("spec/fixtures/dk.png"), "`/png")
+ lfs_object.save!
+ end
end
diff --git a/spec/uploaders/namespace_file_uploader_spec.rb b/spec/uploaders/namespace_file_uploader_spec.rb
index 24a2fc0f72e..a8ba01d70b8 100644
--- a/spec/uploaders/namespace_file_uploader_spec.rb
+++ b/spec/uploaders/namespace_file_uploader_spec.rb
@@ -13,4 +13,26 @@ describe NamespaceFileUploader do
store_dir: %r[uploads/-/system/namespace/\d+],
upload_path: IDENTIFIER,
absolute_path: %r[#{CarrierWave.root}/uploads/-/system/namespace/\d+/#{IDENTIFIER}]
+
+ context "object_store is REMOTE" do
+ before do
+ stub_uploads_object_storage
+ end
+
+ include_context 'with storage', described_class::Store::REMOTE
+
+ it_behaves_like 'builds correct paths',
+ store_dir: %r[namespace/\d+/\h+],
+ upload_path: IDENTIFIER
+ end
+
+ describe "#migrate!" do
+ before do
+ uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/doc_sample.txt')))
+ stub_uploads_object_storage
+ end
+
+ it_behaves_like "migrates", to_store: described_class::Store::REMOTE
+ it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL
+ end
end
diff --git a/spec/uploaders/object_storage_spec.rb b/spec/uploaders/object_storage_spec.rb
new file mode 100644
index 00000000000..489b6707c6e
--- /dev/null
+++ b/spec/uploaders/object_storage_spec.rb
@@ -0,0 +1,326 @@
+require 'rails_helper'
+require 'carrierwave/storage/fog'
+
+class Implementation < GitlabUploader
+ include ObjectStorage::Concern
+ include ::RecordsUploads::Concern
+ prepend ::ObjectStorage::Extension::RecordsUploads
+
+ storage_options Gitlab.config.uploads
+
+ private
+
+ # user/:id
+ def dynamic_segment
+ File.join(model.class.to_s.underscore, model.id.to_s)
+ end
+end
+
+describe ObjectStorage do
+ let(:uploader_class) { Implementation }
+ let(:object) { build_stubbed(:user) }
+ let(:uploader) { uploader_class.new(object, :file) }
+
+ before do
+ allow(uploader_class).to receive(:object_store_enabled?).and_return(true)
+ end
+
+ describe '#object_store=' do
+ it "reload the local storage" do
+ uploader.object_store = described_class::Store::LOCAL
+ expect(uploader.file_storage?).to be_truthy
+ end
+
+ it "reload the REMOTE storage" do
+ uploader.object_store = described_class::Store::REMOTE
+ expect(uploader.file_storage?).to be_falsey
+ end
+ end
+
+ context 'object_store is Store::LOCAL' do
+ before do
+ uploader.object_store = described_class::Store::LOCAL
+ end
+
+ describe '#store_dir' do
+ it 'is the composition of (base_dir, dynamic_segment)' do
+ expect(uploader.store_dir).to start_with("uploads/-/system/user/")
+ end
+ end
+ end
+
+ context 'object_store is Store::REMOTE' do
+ before do
+ uploader.object_store = described_class::Store::REMOTE
+ end
+
+ describe '#store_dir' do
+ it 'is the composition of (dynamic_segment)' do
+ expect(uploader.store_dir).to start_with("user/")
+ end
+ end
+ end
+
+ describe '#object_store' do
+ it "delegates to <mount>_store on model" do
+ expect(object).to receive(:file_store)
+
+ uploader.object_store
+ end
+
+ context 'when store is null' do
+ before do
+ expect(object).to receive(:file_store).and_return(nil)
+ end
+
+ it "returns Store::LOCAL" do
+ expect(uploader.object_store).to eq(described_class::Store::LOCAL)
+ end
+ end
+
+ context 'when value is set' do
+ before do
+ expect(object).to receive(:file_store).and_return(described_class::Store::REMOTE)
+ end
+
+ it "returns the given value" do
+ expect(uploader.object_store).to eq(described_class::Store::REMOTE)
+ end
+ end
+ end
+
+ describe '#file_cache_storage?' do
+ context 'when file storage is used' do
+ before do
+ uploader_class.cache_storage(:file)
+ end
+
+ it { expect(uploader).to be_file_cache_storage }
+ end
+
+ context 'when is remote storage' do
+ before do
+ uploader_class.cache_storage(:fog)
+ end
+
+ it { expect(uploader).not_to be_file_cache_storage }
+ end
+ end
+
+ # this means the model shall include
+ # include RecordsUpload::Concern
+ # prepend ObjectStorage::Extension::RecordsUploads
+ # the object_store persistence is delegated to the `Upload` model.
+ #
+ context 'when persist_object_store? is false' do
+ let(:object) { create(:project, :with_avatar) }
+ let(:uploader) { object.avatar }
+
+ it { expect(object).to be_a(Avatarable) }
+ it { expect(uploader.persist_object_store?).to be_falsey }
+
+ describe 'delegates the object_store logic to the `Upload` model' do
+ it 'sets @upload to the found `upload`' do
+ expect(uploader.upload).to eq(uploader.upload)
+ end
+
+ it 'sets @object_store to the `Upload` value' do
+ expect(uploader.object_store).to eq(uploader.upload.store)
+ end
+ end
+
+ describe '#migrate!' do
+ let(:new_store) { ObjectStorage::Store::REMOTE }
+
+ before do
+ stub_uploads_object_storage(uploader: AvatarUploader)
+ end
+
+ subject { uploader.migrate!(new_store) }
+
+ it 'persist @object_store to the recorded upload' do
+ subject
+
+ expect(uploader.upload.store).to eq(new_store)
+ end
+
+ describe 'fails' do
+ it 'is handled gracefully' do
+ store = uploader.object_store
+ expect_any_instance_of(Upload).to receive(:save!).and_raise("An error")
+
+ expect { subject }.to raise_error("An error")
+ expect(uploader.exists?).to be_truthy
+ expect(uploader.upload.store).to eq(store)
+ end
+ end
+ end
+ end
+
+ # this means the model holds an <mounted_as>_store attribute directly
+ # and do not delegate the object_store persistence to the `Upload` model.
+ #
+ context 'persist_object_store? is true' do
+ context 'when using JobArtifactsUploader' do
+ let(:store) { described_class::Store::LOCAL }
+ let(:object) { create(:ci_job_artifact, :archive, file_store: store) }
+ let(:uploader) { object.file }
+
+ context 'checking described_class' do
+ it "uploader include described_class::Concern" do
+ expect(uploader).to be_a(described_class::Concern)
+ end
+ end
+
+ describe '#use_file' do
+ context 'when file is stored locally' do
+ it "calls a regular path" do
+ expect { |b| uploader.use_file(&b) }.not_to yield_with_args(%r[tmp/cache])
+ end
+ end
+
+ context 'when file is stored remotely' do
+ let(:store) { described_class::Store::REMOTE }
+
+ before do
+ stub_artifacts_object_storage
+ end
+
+ it "calls a cache path" do
+ expect { |b| uploader.use_file(&b) }.to yield_with_args(%r[tmp/cache])
+ end
+ end
+ end
+
+ describe '#migrate!' do
+ subject { uploader.migrate!(new_store) }
+
+ shared_examples "updates the underlying <mounted>_store" do
+ it do
+ subject
+
+ expect(object.file_store).to eq(new_store)
+ end
+ end
+
+ context 'when using the same storage' do
+ let(:new_store) { store }
+
+ it "to not migrate the storage" do
+ subject
+
+ expect(uploader).not_to receive(:store!)
+ expect(uploader.object_store).to eq(store)
+ end
+ end
+
+ context 'when migrating to local storage' do
+ let(:store) { described_class::Store::REMOTE }
+ let(:new_store) { described_class::Store::LOCAL }
+
+ before do
+ stub_artifacts_object_storage
+ end
+
+ include_examples "updates the underlying <mounted>_store"
+
+ it "local file does not exist" do
+ expect(File.exist?(uploader.path)).to eq(false)
+ end
+
+ it "remote file exist" do
+ expect(uploader.file.exists?).to be_truthy
+ end
+
+ it "does migrate the file" do
+ subject
+
+ expect(uploader.object_store).to eq(new_store)
+ expect(File.exist?(uploader.path)).to eq(true)
+ end
+ end
+
+ context 'when migrating to remote storage' do
+ let(:new_store) { described_class::Store::REMOTE }
+ let!(:current_path) { uploader.path }
+
+ it "file does exist" do
+ expect(File.exist?(current_path)).to eq(true)
+ end
+
+ context 'when storage is disabled' do
+ before do
+ stub_artifacts_object_storage(enabled: false)
+ end
+
+ it "to raise an error" do
+ expect { subject }.to raise_error(/Object Storage is not enabled/)
+ end
+ end
+
+ context 'when credentials are set' do
+ before do
+ stub_artifacts_object_storage
+ end
+
+ include_examples "updates the underlying <mounted>_store"
+
+ it "does migrate the file" do
+ subject
+
+ expect(uploader.object_store).to eq(new_store)
+ end
+
+ it "does delete original file" do
+ subject
+
+ expect(File.exist?(current_path)).to eq(false)
+ end
+
+ context 'when subject save fails' do
+ before do
+ expect(uploader).to receive(:persist_object_store!).and_raise(RuntimeError, "exception")
+ end
+
+ it "original file is not removed" do
+ expect { subject }.to raise_error(/exception/)
+
+ expect(File.exist?(current_path)).to eq(true)
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+
+ describe '#fog_directory' do
+ let(:remote_directory) { 'directory' }
+
+ before do
+ uploader_class.storage_options double(object_store: double(remote_directory: remote_directory))
+ end
+
+ subject { uploader.fog_directory }
+
+ it { is_expected.to eq(remote_directory) }
+ end
+
+ describe '#fog_credentials' do
+ let(:connection) { Settingslogic.new("provider" => "AWS") }
+
+ before do
+ uploader_class.storage_options double(object_store: double(connection: connection))
+ end
+
+ subject { uploader.fog_credentials }
+
+ it { is_expected.to eq(provider: 'AWS') }
+ end
+
+ describe '#fog_public' do
+ subject { uploader.fog_public }
+
+ it { is_expected.to eq(false) }
+ end
+end
diff --git a/spec/uploaders/personal_file_uploader_spec.rb b/spec/uploaders/personal_file_uploader_spec.rb
index ed1fba6edda..c70521d90dc 100644
--- a/spec/uploaders/personal_file_uploader_spec.rb
+++ b/spec/uploaders/personal_file_uploader_spec.rb
@@ -14,6 +14,18 @@ describe PersonalFileUploader do
upload_path: IDENTIFIER,
absolute_path: %r[#{CarrierWave.root}/uploads/-/system/personal_snippet/\d+/#{IDENTIFIER}]
+ context "object_store is REMOTE" do
+ before do
+ stub_uploads_object_storage
+ end
+
+ include_context 'with storage', described_class::Store::REMOTE
+
+ it_behaves_like 'builds correct paths',
+ store_dir: %r[\d+/\h+],
+ upload_path: IDENTIFIER
+ end
+
describe '#to_h' do
before do
subject.instance_variable_set(:@secret, 'secret')
@@ -30,4 +42,14 @@ describe PersonalFileUploader do
)
end
end
+
+ describe "#migrate!" do
+ before do
+ uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/doc_sample.txt')))
+ stub_uploads_object_storage
+ end
+
+ it_behaves_like "migrates", to_store: described_class::Store::REMOTE
+ it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL
+ end
end
diff --git a/spec/workers/object_storage_upload_worker_spec.rb b/spec/workers/object_storage_upload_worker_spec.rb
new file mode 100644
index 00000000000..32ddcbe9757
--- /dev/null
+++ b/spec/workers/object_storage_upload_worker_spec.rb
@@ -0,0 +1,108 @@
+require 'spec_helper'
+
+describe ObjectStorageUploadWorker do
+ let(:local) { ObjectStorage::Store::LOCAL }
+ let(:remote) { ObjectStorage::Store::REMOTE }
+
+ def perform
+ described_class.perform_async(uploader_class.name, subject_class, file_field, subject_id)
+ end
+
+ context 'for LFS' do
+ let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) }
+ let(:uploader_class) { LfsObjectUploader }
+ let(:subject_class) { LfsObject }
+ let(:file_field) { :file }
+ let(:subject_id) { lfs_object.id }
+
+ context 'when object storage is enabled' do
+ before do
+ stub_lfs_object_storage(background_upload: true)
+ end
+
+ it 'uploads object to storage' do
+ expect { perform }.to change { lfs_object.reload.file_store }.from(local).to(remote)
+ end
+
+ context 'when background upload is disabled' do
+ before do
+ allow(Gitlab.config.lfs.object_store).to receive(:background_upload) { false }
+ end
+
+ it 'is skipped' do
+ expect { perform }.not_to change { lfs_object.reload.file_store }
+ end
+ end
+ end
+
+ context 'when object storage is disabled' do
+ before do
+ stub_lfs_object_storage(enabled: false)
+ end
+
+ it "doesn't migrate files" do
+ perform
+
+ expect(lfs_object.reload.file_store).to eq(local)
+ end
+ end
+ end
+
+ context 'for legacy artifacts' do
+ let(:build) { create(:ci_build, :legacy_artifacts) }
+ let(:uploader_class) { LegacyArtifactUploader }
+ let(:subject_class) { Ci::Build }
+ let(:file_field) { :artifacts_file }
+ let(:subject_id) { build.id }
+
+ context 'when local storage is used' do
+ let(:store) { local }
+
+ context 'and remote storage is defined' do
+ before do
+ stub_artifacts_object_storage(background_upload: true)
+ end
+
+ it "migrates file to remote storage" do
+ perform
+
+ expect(build.reload.artifacts_file_store).to eq(remote)
+ end
+
+ context 'for artifacts_metadata' do
+ let(:file_field) { :artifacts_metadata }
+
+ it 'migrates metadata to remote storage' do
+ perform
+
+ expect(build.reload.artifacts_metadata_store).to eq(remote)
+ end
+ end
+ end
+ end
+ end
+
+ context 'for job artifacts' do
+ let(:artifact) { create(:ci_job_artifact, :archive) }
+ let(:uploader_class) { JobArtifactUploader }
+ let(:subject_class) { Ci::JobArtifact }
+ let(:file_field) { :file }
+ let(:subject_id) { artifact.id }
+
+ context 'when local storage is used' do
+ let(:store) { local }
+
+ context 'and remote storage is defined' do
+ before do
+ stub_artifacts_object_storage(background_upload: true)
+ end
+
+ it "migrates file to remote storage" do
+ perform
+
+ expect(artifact.reload.file_store).to eq(remote)
+ end
+ end
+ end
+ end
+end