diff options
author | Grzegorz Bizon <grzegorz@gitlab.com> | 2018-04-05 20:59:40 +0000 |
---|---|---|
committer | Grzegorz Bizon <grzegorz@gitlab.com> | 2018-04-05 20:59:40 +0000 |
commit | dd271e246001a06609592eef109d154291305d32 (patch) | |
tree | 279ef8877a1a7507d26c042810410f6362114e35 /spec | |
parent | e9e800f5239f5b45984d49615bbc67e823a117ab (diff) | |
parent | 678620cce67cc283b19b75137f747f9415aaf942 (diff) | |
download | gitlab-ce-dd271e246001a06609592eef109d154291305d32.tar.gz |
Merge branch 'direct-upload-of-artifacts' into 'master'
Direct upload of artifacts
See merge request gitlab-org/gitlab-ce!18160
Diffstat (limited to 'spec')
-rw-r--r-- | spec/initializers/artifacts_direct_upload_support_spec.rb | 71 | ||||
-rw-r--r-- | spec/lib/gitlab/import_export/project.json | 60 | ||||
-rw-r--r-- | spec/lib/uploaded_file_spec.rb | 116 | ||||
-rw-r--r-- | spec/requests/api/runner_spec.rb | 117 | ||||
-rw-r--r-- | spec/requests/lfs_http_spec.rb | 19 | ||||
-rw-r--r-- | spec/support/stub_configuration.rb | 4 | ||||
-rw-r--r-- | spec/uploaders/object_storage_spec.rb | 155 |
7 files changed, 341 insertions, 201 deletions
diff --git a/spec/initializers/artifacts_direct_upload_support_spec.rb b/spec/initializers/artifacts_direct_upload_support_spec.rb new file mode 100644 index 00000000000..bfb71da3388 --- /dev/null +++ b/spec/initializers/artifacts_direct_upload_support_spec.rb @@ -0,0 +1,71 @@ +require 'spec_helper' + +describe 'Artifacts direct upload support' do + subject do + load Rails.root.join('config/initializers/artifacts_direct_upload_support.rb') + end + + let(:connection) do + { provider: provider } + end + + before do + stub_artifacts_setting( + object_store: { + enabled: enabled, + direct_upload: direct_upload, + connection: connection + }) + end + + context 'when object storage is enabled' do + let(:enabled) { true } + + context 'when direct upload is enabled' do + let(:direct_upload) { true } + + context 'when provider is Google' do + let(:provider) { 'Google' } + + it 'succeeds' do + expect { subject }.not_to raise_error + end + end + + context 'when connection is empty' do + let(:connection) { nil } + + it 'raises an error' do + expect { subject }.to raise_error /object storage provider when 'direct_upload' of artifacts is used/ + end + end + + context 'when other provider is used' do + let(:provider) { 'AWS' } + + it 'raises an error' do + expect { subject }.to raise_error /object storage provider when 'direct_upload' of artifacts is used/ + end + end + end + + context 'when direct upload is disabled' do + let(:direct_upload) { false } + let(:provider) { 'AWS' } + + it 'succeeds' do + expect { subject }.not_to raise_error + end + end + end + + context 'when object storage is disabled' do + let(:enabled) { false } + let(:direct_upload) { false } + let(:provider) { 'AWS' } + + it 'succeeds' do + expect { subject }.not_to raise_error + end + end +end diff --git a/spec/lib/gitlab/import_export/project.json b/spec/lib/gitlab/import_export/project.json index 8c1bf6f7be8..6d63749296e 100644 --- a/spec/lib/gitlab/import_export/project.json +++ b/spec/lib/gitlab/import_export/project.json @@ -6180,12 +6180,6 @@ "user_id": null, "target_url": null, "description": null, - "artifacts_file": { - "url": null - }, - "artifacts_metadata": { - "url": null - }, "erased_by_id": null, "erased_at": null, "type": "Ci::Build", @@ -6218,12 +6212,6 @@ "user_id": null, "target_url": null, "description": null, - "artifacts_file": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/72/p5_build_artifacts.zip" - }, - "artifacts_metadata": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/72/p5_build_artifacts_metadata.gz" - }, "erased_by_id": null, "erased_at": null } @@ -6292,12 +6280,6 @@ "user_id": null, "target_url": null, "description": null, - "artifacts_file": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/74/p5_build_artifacts.zip" - }, - "artifacts_metadata": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/74/p5_build_artifacts_metadata.gz" - }, "erased_by_id": null, "erased_at": null }, @@ -6327,12 +6309,6 @@ "user_id": null, "target_url": null, "description": null, - "artifacts_file": { - "url": null - }, - "artifacts_metadata": { - "url": null - }, "erased_by_id": null, "erased_at": null } @@ -6392,12 +6368,6 @@ "user_id": null, "target_url": null, "description": null, - "artifacts_file": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/76/p5_build_artifacts.zip" - }, - "artifacts_metadata": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/76/p5_build_artifacts_metadata.gz" - }, "erased_by_id": null, "erased_at": null }, @@ -6427,12 +6397,6 @@ "user_id": null, "target_url": null, "description": null, - "artifacts_file": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/75/p5_build_artifacts.zip" - }, - "artifacts_metadata": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/75/p5_build_artifacts_metadata.gz" - }, "erased_by_id": null, "erased_at": null } @@ -6492,12 +6456,6 @@ "user_id": null, "target_url": null, "description": null, - "artifacts_file": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/78/p5_build_artifacts.zip" - }, - "artifacts_metadata": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/78/p5_build_artifacts_metadata.gz" - }, "erased_by_id": null, "erased_at": null }, @@ -6527,12 +6485,6 @@ "user_id": null, "target_url": null, "description": null, - "artifacts_file": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/77/p5_build_artifacts.zip" - }, - "artifacts_metadata": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/77/p5_build_artifacts_metadata.gz" - }, "erased_by_id": null, "erased_at": null } @@ -6592,12 +6544,6 @@ "user_id": null, "target_url": null, "description": null, - "artifacts_file": { - "url": null - }, - "artifacts_metadata": { - "url": null - }, "erased_by_id": null, "erased_at": null }, @@ -6627,12 +6573,6 @@ "user_id": null, "target_url": null, "description": null, - "artifacts_file": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/80/p5_build_artifacts.zip" - }, - "artifacts_metadata": { - "url": "/Users/Test/Test/gitlab-development-kit/gitlab/shared/artifacts/2016_03/5/80/p5_build_artifacts_metadata.gz" - }, "erased_by_id": null, "erased_at": null } diff --git a/spec/lib/uploaded_file_spec.rb b/spec/lib/uploaded_file_spec.rb new file mode 100644 index 00000000000..cc99e7e8911 --- /dev/null +++ b/spec/lib/uploaded_file_spec.rb @@ -0,0 +1,116 @@ +require 'spec_helper' + +describe UploadedFile do + describe ".from_params" do + let(:temp_dir) { Dir.tmpdir } + let(:temp_file) { Tempfile.new("test", temp_dir) } + let(:upload_path) { nil } + + subject do + described_class.from_params(params, :file, upload_path) + end + + before do + FileUtils.touch(temp_file) + end + + after do + FileUtils.rm_f(temp_file) + FileUtils.rm_r(upload_path) if upload_path + end + + context 'when valid file is specified' do + context 'only local path is specified' do + let(:params) do + { 'file.path' => temp_file.path } + end + + it "succeeds" do + is_expected.not_to be_nil + end + + it "generates filename from path" do + expect(subject.original_filename).to eq(::File.basename(temp_file.path)) + end + end + + context 'all parameters are specified' do + let(:params) do + { 'file.path' => temp_file.path, + 'file.name' => 'my_file.txt', + 'file.type' => 'my/type', + 'file.sha256' => 'sha256', + 'file.remote_id' => 'remote_id' } + end + + it "succeeds" do + is_expected.not_to be_nil + end + + it "generates filename from path" do + expect(subject.original_filename).to eq('my_file.txt') + expect(subject.content_type).to eq('my/type') + expect(subject.sha256).to eq('sha256') + expect(subject.remote_id).to eq('remote_id') + end + end + end + + context 'when no params are specified' do + let(:params) do + {} + end + + it "does not return an object" do + is_expected.to be_nil + end + end + + context 'when only remote id is specified' do + let(:params) do + { 'file.remote_id' => 'remote_id' } + end + + it "raises an error" do + expect { subject }.to raise_error(UploadedFile::InvalidPathError, /file is invalid/) + end + end + + context 'when verifying allowed paths' do + let(:params) do + { 'file.path' => temp_file.path } + end + + context 'when file is stored in system temporary folder' do + let(:temp_dir) { Dir.tmpdir } + + it "succeeds" do + is_expected.not_to be_nil + end + end + + context 'when file is stored in user provided upload path' do + let(:upload_path) { Dir.mktmpdir } + let(:temp_dir) { upload_path } + + it "succeeds" do + is_expected.not_to be_nil + end + end + + context 'when file is stored outside of user provided upload path' do + let!(:generated_dir) { Dir.mktmpdir } + let!(:temp_dir) { Dir.mktmpdir } + + before do + # We overwrite default temporary path + allow(Dir).to receive(:tmpdir).and_return(generated_dir) + end + + it "raises an error" do + expect { subject }.to raise_error(UploadedFile::InvalidPathError, /insecure path used/) + end + end + end + end +end diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb index 4f3420cc0ad..28d51ac86c6 100644 --- a/spec/requests/api/runner_spec.rb +++ b/spec/requests/api/runner_spec.rb @@ -950,12 +950,53 @@ describe API::Runner do describe 'POST /api/v4/jobs/:id/artifacts/authorize' do context 'when using token as parameter' do - it 'authorizes posting artifacts to running job' do - authorize_artifacts_with_token_in_params + context 'posting artifacts to running job' do + subject do + authorize_artifacts_with_token_in_params + end - expect(response).to have_gitlab_http_status(200) - expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) - expect(json_response['TempPath']).not_to be_nil + shared_examples 'authorizes local file' do + it 'succeeds' do + subject + + expect(response).to have_gitlab_http_status(200) + expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) + expect(json_response['TempPath']).to eq(JobArtifactUploader.workhorse_local_upload_path) + expect(json_response['RemoteObject']).to be_nil + end + end + + context 'when using local storage' do + it_behaves_like 'authorizes local file' + end + + context 'when using remote storage' do + context 'when direct upload is enabled' do + before do + stub_artifacts_object_storage(enabled: true, direct_upload: true) + end + + it 'succeeds' do + subject + + expect(response).to have_gitlab_http_status(200) + expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) + expect(json_response['TempPath']).to eq(JobArtifactUploader.workhorse_local_upload_path) + expect(json_response['RemoteObject']).to have_key('ID') + expect(json_response['RemoteObject']).to have_key('GetURL') + expect(json_response['RemoteObject']).to have_key('StoreURL') + expect(json_response['RemoteObject']).to have_key('DeleteURL') + end + end + + context 'when direct upload is disabled' do + before do + stub_artifacts_object_storage(enabled: true, direct_upload: false) + end + + it_behaves_like 'authorizes local file' + end + end end it 'fails to post too large artifact' do @@ -1051,20 +1092,45 @@ describe API::Runner do end end - context 'when uses regular file post' do - before do - upload_artifacts(file_upload, headers_with_token, false) + context 'when uses accelerated file post' do + context 'for file stored locally' do + before do + upload_artifacts(file_upload, headers_with_token) + end + + it_behaves_like 'successful artifacts upload' end - it_behaves_like 'successful artifacts upload' - end + context 'for file stored remotelly' do + let!(:fog_connection) do + stub_artifacts_object_storage(direct_upload: true) + end - context 'when uses accelerated file post' do - before do - upload_artifacts(file_upload, headers_with_token, true) - end + before do + fog_connection.directories.get('artifacts').files.create( + key: 'tmp/upload/12312300', + body: 'content' + ) - it_behaves_like 'successful artifacts upload' + upload_artifacts(file_upload, headers_with_token, + { 'file.remote_id' => remote_id }) + end + + context 'when valid remote_id is used' do + let(:remote_id) { '12312300' } + + it_behaves_like 'successful artifacts upload' + end + + context 'when invalid remote_id is used' do + let(:remote_id) { 'invalid id' } + + it 'responds with bad request' do + expect(response).to have_gitlab_http_status(500) + expect(json_response['message']).to eq("Missing file") + end + end + end end context 'when using runners token' do @@ -1208,15 +1274,19 @@ describe API::Runner do end context 'when artifacts are being stored outside of tmp path' do + let(:new_tmpdir) { Dir.mktmpdir } + before do + # init before overwriting tmp dir + file_upload + # by configuring this path we allow to pass file from @tmpdir only # but all temporary files are stored in system tmp directory - @tmpdir = Dir.mktmpdir - allow(JobArtifactUploader).to receive(:workhorse_upload_path).and_return(@tmpdir) + allow(Dir).to receive(:tmpdir).and_return(new_tmpdir) end after do - FileUtils.remove_entry @tmpdir + FileUtils.remove_entry(new_tmpdir) end it' "fails to post artifacts for outside of tmp path"' do @@ -1226,12 +1296,11 @@ describe API::Runner do end end - def upload_artifacts(file, headers = {}, accelerated = true) - params = if accelerated - { 'file.path' => file.path, 'file.name' => file.original_filename } - else - { 'file' => file } - end + def upload_artifacts(file, headers = {}, params = {}) + params = params.merge({ + 'file.path' => file.path, + 'file.name' => file.original_filename + }) post api("/jobs/#{job.id}/artifacts"), params, headers end diff --git a/spec/requests/lfs_http_spec.rb b/spec/requests/lfs_http_spec.rb index 1e6bd993c08..f80abb06fca 100644 --- a/spec/requests/lfs_http_spec.rb +++ b/spec/requests/lfs_http_spec.rb @@ -1016,7 +1016,7 @@ describe 'Git LFS API and storage' do it_behaves_like 'a valid response' do it 'responds with status 200, location of lfs remote store and object details' do - expect(json_response['TempPath']).to be_nil + expect(json_response['TempPath']).to eq(LfsObjectUploader.workhorse_local_upload_path) expect(json_response['RemoteObject']).to have_key('ID') expect(json_response['RemoteObject']).to have_key('GetURL') expect(json_response['RemoteObject']).to have_key('StoreURL') @@ -1073,7 +1073,9 @@ describe 'Git LFS API and storage' do ['123123', '../../123123'].each do |remote_id| context "with invalid remote_id: #{remote_id}" do subject do - put_finalize_with_args('file.remote_id' => remote_id) + put_finalize(with_tempfile: true, args: { + 'file.remote_id' => remote_id + }) end it 'responds with status 403' do @@ -1093,9 +1095,10 @@ describe 'Git LFS API and storage' do end subject do - put_finalize_with_args( + put_finalize(with_tempfile: true, args: { 'file.remote_id' => '12312300', - 'file.name' => 'name') + 'file.name' => 'name' + }) end it 'responds with status 200' do @@ -1331,7 +1334,7 @@ describe 'Git LFS API and storage' do put "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}/authorize", nil, authorize_headers end - def put_finalize(lfs_tmp = lfs_tmp_file, with_tempfile: false) + def put_finalize(lfs_tmp = lfs_tmp_file, with_tempfile: false, args: {}) upload_path = LfsObjectUploader.workhorse_local_upload_path file_path = upload_path + '/' + lfs_tmp if lfs_tmp @@ -1340,12 +1343,12 @@ describe 'Git LFS API and storage' do FileUtils.touch(file_path) end - args = { + extra_args = { 'file.path' => file_path, 'file.name' => File.basename(file_path) - }.compact + } - put_finalize_with_args(args) + put_finalize_with_args(args.merge(extra_args).compact) end def put_finalize_with_args(args) diff --git a/spec/support/stub_configuration.rb b/spec/support/stub_configuration.rb index bad1d34df3a..a75a3eaefcb 100644 --- a/spec/support/stub_configuration.rb +++ b/spec/support/stub_configuration.rb @@ -45,6 +45,10 @@ module StubConfiguration allow(Gitlab.config.lfs).to receive_messages(to_settings(messages)) end + def stub_artifacts_setting(messages) + allow(Gitlab.config.artifacts).to receive_messages(to_settings(messages)) + end + def stub_storage_settings(messages) messages.deep_stringify_keys! diff --git a/spec/uploaders/object_storage_spec.rb b/spec/uploaders/object_storage_spec.rb index 17d508d0146..16455e2517b 100644 --- a/spec/uploaders/object_storage_spec.rb +++ b/spec/uploaders/object_storage_spec.rb @@ -516,108 +516,46 @@ describe ObjectStorage do end end - describe '#store_workhorse_file!' do + describe '#cache!' do subject do - uploader.store_workhorse_file!(params, :file) + uploader.cache!(uploaded_file) end context 'when local file is used' do context 'when valid file is used' do - let(:target_path) do - File.join(uploader_class.root, uploader_class::TMP_UPLOAD_PATH) + let(:uploaded_file) do + fixture_file_upload(Rails.root + 'spec/fixtures/rails_sample.jpg', 'image/jpg') end - before do - FileUtils.mkdir_p(target_path) - end - - context 'when no filename is specified' do - let(:params) do - { "file.path" => "test/file" } - end - - it 'raises an error' do - expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Missing filename/) - end - end - - context 'when invalid file is specified' do - let(:file_path) do - File.join(target_path, "..", "test.file") - end - - before do - FileUtils.touch(file_path) - end - - let(:params) do - { "file.path" => file_path, - "file.name" => "my_file.txt" } - end - - it 'raises an error' do - expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Bad file path/) - end - end - - context 'when filename is specified' do - let(:params) do - { "file.path" => tmp_file, - "file.name" => "my_file.txt" } - end - - let(:tmp_file) { Tempfile.new('filename', target_path) } - - before do - FileUtils.touch(tmp_file) - end - - after do - FileUtils.rm_f(tmp_file) - end - - it 'succeeds' do - expect { subject }.not_to raise_error - - expect(uploader).to be_exists - end - - it 'proper path is being used' do - subject - - expect(uploader.path).to start_with(uploader_class.root) - expect(uploader.path).to end_with("my_file.txt") - end + it "properly caches the file" do + subject - it 'source file to not exist' do - subject - - expect(File.exist?(tmp_file.path)).to be_falsey - end + expect(uploader).to be_exists + expect(uploader.path).to start_with(uploader_class.root) + expect(uploader.filename).to eq('rails_sample.jpg') end end end context 'when remote file is used' do + let(:temp_file) { Tempfile.new("test") } + let!(:fog_connection) do stub_uploads_object_storage(uploader_class) end - context 'when valid file is used' do - context 'when no filename is specified' do - let(:params) do - { "file.remote_id" => "test/123123" } - end + before do + FileUtils.touch(temp_file) + end - it 'raises an error' do - expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Missing filename/) - end - end + after do + FileUtils.rm_f(temp_file) + end + context 'when valid file is used' do context 'when invalid file is specified' do - let(:params) do - { "file.remote_id" => "../test/123123", - "file.name" => "my_file.txt" } + let(:uploaded_file) do + UploadedFile.new(temp_file.path, remote_id: "../test/123123") end it 'raises an error' do @@ -626,9 +564,8 @@ describe ObjectStorage do end context 'when non existing file is specified' do - let(:params) do - { "file.remote_id" => "test/12312300", - "file.name" => "my_file.txt" } + let(:uploaded_file) do + UploadedFile.new(temp_file.path, remote_id: "test/123123") end it 'raises an error' do @@ -636,10 +573,9 @@ describe ObjectStorage do end end - context 'when filename is specified' do - let(:params) do - { "file.remote_id" => "test/123123", - "file.name" => "my_file.txt" } + context 'when valid file is specified' do + let(:uploaded_file) do + UploadedFile.new(temp_file.path, filename: "my_file.txt", remote_id: "test/123123") end let!(:fog_file) do @@ -649,36 +585,37 @@ describe ObjectStorage do ) end - it 'succeeds' do + it 'file to be cached and remote stored' do expect { subject }.not_to raise_error expect(uploader).to be_exists - end - - it 'path to not be temporary' do - subject - + expect(uploader).to be_cached expect(uploader.path).not_to be_nil - expect(uploader.path).not_to include('tmp/upload') - expect(uploader.url).to include('/my_file.txt') + expect(uploader.path).not_to include('tmp/cache') + expect(uploader.url).not_to be_nil + expect(uploader.path).not_to include('tmp/cache') + expect(uploader.object_store).to eq(described_class::Store::REMOTE) end - it 'url is used' do - subject + context 'when file is stored' do + subject do + uploader.store!(uploaded_file) + end - expect(uploader.url).not_to be_nil - expect(uploader.url).to include('/my_file.txt') + it 'file to be remotely stored in permament location' do + subject + + expect(uploader).to be_exists + expect(uploader).not_to be_cached + expect(uploader.path).not_to be_nil + expect(uploader.path).not_to include('tmp/upload') + expect(uploader.path).not_to include('tmp/cache') + expect(uploader.url).to include('/my_file.txt') + expect(uploader.object_store).to eq(described_class::Store::REMOTE) + end end end end end - - context 'when no file is used' do - let(:params) { {} } - - it 'raises an error' do - expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Bad file/) - end - end end end |