diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 16:05:49 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 16:05:49 +0000 |
commit | 43a25d93ebdabea52f99b05e15b06250cd8f07d7 (patch) | |
tree | dceebdc68925362117480a5d672bcff122fb625b /spec/uploaders | |
parent | 20c84b99005abd1c82101dfeff264ac50d2df211 (diff) | |
download | gitlab-ce-0f94cf6ca9d272d8e0fda4a7a597866cf3dc1fc0.tar.gz |
Add latest changes from gitlab-org/gitlab@16-0-stable-eev16.0.0-rc4216-0-stable
Diffstat (limited to 'spec/uploaders')
21 files changed, 476 insertions, 164 deletions
diff --git a/spec/uploaders/attachment_uploader_spec.rb b/spec/uploaders/attachment_uploader_spec.rb index 05cffff1f1a..a035402e207 100644 --- a/spec/uploaders/attachment_uploader_spec.rb +++ b/spec/uploaders/attachment_uploader_spec.rb @@ -10,9 +10,9 @@ RSpec.describe AttachmentUploader do subject { uploader } it_behaves_like 'builds correct paths', - store_dir: %r[uploads/-/system/note/attachment/], - upload_path: %r[uploads/-/system/note/attachment/], - absolute_path: %r[#{CarrierWave.root}/uploads/-/system/note/attachment/] + store_dir: %r[uploads/-/system/note/attachment/], + upload_path: %r[uploads/-/system/note/attachment/], + absolute_path: %r[#{CarrierWave.root}/uploads/-/system/note/attachment/] context "object_store is REMOTE" do before do @@ -22,8 +22,8 @@ RSpec.describe AttachmentUploader do include_context 'with storage', described_class::Store::REMOTE it_behaves_like 'builds correct paths', - store_dir: %r[note/attachment/], - upload_path: %r[note/attachment/] + store_dir: %r[note/attachment/], + upload_path: %r[note/attachment/] end describe "#migrate!" do diff --git a/spec/uploaders/avatar_uploader_spec.rb b/spec/uploaders/avatar_uploader_spec.rb index a55e5c23fe8..e472ac46e66 100644 --- a/spec/uploaders/avatar_uploader_spec.rb +++ b/spec/uploaders/avatar_uploader_spec.rb @@ -10,9 +10,9 @@ RSpec.describe AvatarUploader do subject { uploader } it_behaves_like 'builds correct paths', - store_dir: %r[uploads/-/system/user/avatar/], - upload_path: %r[uploads/-/system/user/avatar/], - absolute_path: %r[#{CarrierWave.root}/uploads/-/system/user/avatar/] + store_dir: %r[uploads/-/system/user/avatar/], + upload_path: %r[uploads/-/system/user/avatar/], + absolute_path: %r[#{CarrierWave.root}/uploads/-/system/user/avatar/] context "object_store is REMOTE" do before do @@ -22,8 +22,8 @@ RSpec.describe AvatarUploader do include_context 'with storage', described_class::Store::REMOTE it_behaves_like 'builds correct paths', - store_dir: %r[user/avatar/], - upload_path: %r[user/avatar/] + store_dir: %r[user/avatar/], + upload_path: %r[user/avatar/] end context "with a file" do diff --git a/spec/uploaders/ci/pipeline_artifact_uploader_spec.rb b/spec/uploaders/ci/pipeline_artifact_uploader_spec.rb index 0630e9f6546..3935f081372 100644 --- a/spec/uploaders/ci/pipeline_artifact_uploader_spec.rb +++ b/spec/uploaders/ci/pipeline_artifact_uploader_spec.rb @@ -9,9 +9,9 @@ RSpec.describe Ci::PipelineArtifactUploader do subject { uploader } it_behaves_like "builds correct paths", - store_dir: %r[\h{2}/\h{2}/\h{64}/pipelines/\d+/artifacts/\d+], - cache_dir: %r[artifacts/tmp/cache], - work_dir: %r[artifacts/tmp/work] + store_dir: %r[\h{2}/\h{2}/\h{64}/pipelines/\d+/artifacts/\d+], + cache_dir: %r[artifacts/tmp/cache], + work_dir: %r[artifacts/tmp/work] context 'when object store is REMOTE' do before do diff --git a/spec/uploaders/dependency_proxy/file_uploader_spec.rb b/spec/uploaders/dependency_proxy/file_uploader_spec.rb index eb12e7dffa5..3cb2d1ea0f0 100644 --- a/spec/uploaders/dependency_proxy/file_uploader_spec.rb +++ b/spec/uploaders/dependency_proxy/file_uploader_spec.rb @@ -11,9 +11,9 @@ RSpec.describe DependencyProxy::FileUploader do subject { uploader } it_behaves_like "builds correct paths", - store_dir: %r[\h{2}/\h{2}], - cache_dir: %r[/dependency_proxy/tmp/cache], - work_dir: %r[/dependency_proxy/tmp/work] + store_dir: %r[\h{2}/\h{2}], + cache_dir: %r[/dependency_proxy/tmp/cache], + work_dir: %r[/dependency_proxy/tmp/work] context 'object store is remote' do before do @@ -22,8 +22,7 @@ RSpec.describe DependencyProxy::FileUploader do include_context 'with storage', described_class::Store::REMOTE - it_behaves_like "builds correct paths", - store_dir: %r[\h{2}/\h{2}] + it_behaves_like "builds correct paths", store_dir: %r[\h{2}/\h{2}] end end diff --git a/spec/uploaders/design_management/design_v432x230_uploader_spec.rb b/spec/uploaders/design_management/design_v432x230_uploader_spec.rb index a18a37e73da..f3dd77d67a0 100644 --- a/spec/uploaders/design_management/design_v432x230_uploader_spec.rb +++ b/spec/uploaders/design_management/design_v432x230_uploader_spec.rb @@ -11,10 +11,10 @@ RSpec.describe DesignManagement::DesignV432x230Uploader do subject(:uploader) { described_class.new(model, :image_v432x230) } it_behaves_like 'builds correct paths', - store_dir: %r[uploads/-/system/design_management/action/image_v432x230/], - upload_path: %r[uploads/-/system/design_management/action/image_v432x230/], - relative_path: %r[uploads/-/system/design_management/action/image_v432x230/], - absolute_path: %r[#{CarrierWave.root}/uploads/-/system/design_management/action/image_v432x230/] + store_dir: %r[uploads/-/system/design_management/action/image_v432x230/], + upload_path: %r[uploads/-/system/design_management/action/image_v432x230/], + relative_path: %r[uploads/-/system/design_management/action/image_v432x230/], + absolute_path: %r[#{CarrierWave.root}/uploads/-/system/design_management/action/image_v432x230/] context 'object_store is REMOTE' do before do @@ -24,9 +24,9 @@ RSpec.describe DesignManagement::DesignV432x230Uploader do include_context 'with storage', described_class::Store::REMOTE it_behaves_like 'builds correct paths', - store_dir: %r[design_management/action/image_v432x230/], - upload_path: %r[design_management/action/image_v432x230/], - relative_path: %r[design_management/action/image_v432x230/] + store_dir: %r[design_management/action/image_v432x230/], + upload_path: %r[design_management/action/image_v432x230/], + relative_path: %r[design_management/action/image_v432x230/] end describe "#migrate!" do diff --git a/spec/uploaders/external_diff_uploader_spec.rb b/spec/uploaders/external_diff_uploader_spec.rb index a889181b72c..2121e9cbc29 100644 --- a/spec/uploaders/external_diff_uploader_spec.rb +++ b/spec/uploaders/external_diff_uploader_spec.rb @@ -9,9 +9,9 @@ RSpec.describe ExternalDiffUploader do subject(:uploader) { described_class.new(diff, :external_diff) } it_behaves_like "builds correct paths", - store_dir: %r[merge_request_diffs/mr-\d+], - cache_dir: %r[/external-diffs/tmp/cache], - work_dir: %r[/external-diffs/tmp/work] + store_dir: %r[merge_request_diffs/mr-\d+], + cache_dir: %r[/external-diffs/tmp/cache], + work_dir: %r[/external-diffs/tmp/work] context "object store is REMOTE" do before do @@ -21,7 +21,7 @@ RSpec.describe ExternalDiffUploader do include_context 'with storage', described_class::Store::REMOTE it_behaves_like "builds correct paths", - store_dir: %r[merge_request_diffs/mr-\d+] + store_dir: %r[merge_request_diffs/mr-\d+] end describe 'remote file' do diff --git a/spec/uploaders/file_uploader_spec.rb b/spec/uploaders/file_uploader_spec.rb index 1287b809223..3340725dd6d 100644 --- a/spec/uploaders/file_uploader_spec.rb +++ b/spec/uploaders/file_uploader_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' RSpec.describe FileUploader do - let(:group) { create(:group, name: 'awesome') } - let(:project) { create(:project, :legacy_storage, namespace: group, name: 'project') } + let(:group) { create(:group, path: 'awesome') } + let(:project) { create(:project, :legacy_storage, namespace: group, path: 'project') } let(:uploader) { described_class.new(project, :avatar) } let(:upload) { double(model: project, path: "#{secret}/foo.jpg") } let(:secret) { "55dc16aa0edd05693fd98b5051e83321" } # this would be nicer as SecureRandom.hex, but the shared_examples breaks @@ -13,9 +13,9 @@ RSpec.describe FileUploader do shared_examples 'builds correct legacy storage paths' do include_examples 'builds correct paths', - store_dir: %r{awesome/project/\h+}, - upload_path: %r{\h+/<filename>}, - absolute_path: %r{#{described_class.root}/awesome/project/55dc16aa0edd05693fd98b5051e83321/foo.jpg} + store_dir: %r{awesome/project/\h+}, + upload_path: %r{\h+/<filename>}, + absolute_path: %r{#{described_class.root}/awesome/project/55dc16aa0edd05693fd98b5051e83321/foo.jpg} end context 'legacy storage' do @@ -23,15 +23,15 @@ RSpec.describe FileUploader do context 'uses hashed storage' do context 'when rolled out attachments' do - let(:project) { build_stubbed(:project, namespace: group, name: 'project') } + let(:project) { build_stubbed(:project, namespace: group, path: 'project') } include_examples 'builds correct paths', - store_dir: %r{@hashed/\h{2}/\h{2}/\h+}, - upload_path: %r{\h+/<filename>} + store_dir: %r{@hashed/\h{2}/\h{2}/\h+}, + upload_path: %r{\h+/<filename>} end context 'when only repositories are rolled out' do - let(:project) { build_stubbed(:project, namespace: group, name: 'project', storage_version: Project::HASHED_STORAGE_FEATURES[:repository]) } + let(:project) { build_stubbed(:project, namespace: group, path: 'project', storage_version: Project::HASHED_STORAGE_FEATURES[:repository]) } it_behaves_like 'builds correct legacy storage paths' end @@ -47,8 +47,8 @@ RSpec.describe FileUploader do # always use hashed storage path for remote uploads it_behaves_like 'builds correct paths', - store_dir: %r{@hashed/\h{2}/\h{2}/\h+}, - upload_path: %r{@hashed/\h{2}/\h{2}/\h+/\h+/<filename>} + store_dir: %r{@hashed/\h{2}/\h{2}/\h+}, + upload_path: %r{@hashed/\h{2}/\h{2}/\h+/\h+/<filename>} end describe 'initialize' do diff --git a/spec/uploaders/gitlab_uploader_spec.rb b/spec/uploaders/gitlab_uploader_spec.rb index f62ab726631..bd86f1fe08a 100644 --- a/spec/uploaders/gitlab_uploader_spec.rb +++ b/spec/uploaders/gitlab_uploader_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' require 'carrierwave/storage/fog' -RSpec.describe GitlabUploader do +RSpec.describe GitlabUploader, feature_category: :shared do let(:uploader_class) { Class.new(described_class) } subject(:uploader) { uploader_class.new(double) } @@ -179,4 +179,19 @@ RSpec.describe GitlabUploader do it { expect { subject }.to raise_error(RuntimeError, /not supported/) } end + + describe '.storage_location' do + it 'sets the identifier for the storage location options' do + uploader_class.storage_location(:artifacts) + + expect(uploader_class.options).to eq(Gitlab.config.artifacts) + end + + context 'when given identifier is not known' do + it 'raises an error' do + expect { uploader_class.storage_location(:foo) } + .to raise_error(KeyError) + end + end + end end diff --git a/spec/uploaders/job_artifact_uploader_spec.rb b/spec/uploaders/job_artifact_uploader_spec.rb index d7c9ef7e0d5..dac9e97641d 100644 --- a/spec/uploaders/job_artifact_uploader_spec.rb +++ b/spec/uploaders/job_artifact_uploader_spec.rb @@ -10,9 +10,9 @@ RSpec.describe JobArtifactUploader do subject { uploader } it_behaves_like "builds correct paths", - store_dir: %r[\h{2}/\h{2}/\h{64}/\d{4}_\d{1,2}_\d{1,2}/\d+/\d+\z], - cache_dir: %r[artifacts/tmp/cache], - work_dir: %r[artifacts/tmp/work] + store_dir: %r[\h{2}/\h{2}/\h{64}/\d{4}_\d{1,2}_\d{1,2}/\d+/\d+\z], + cache_dir: %r[artifacts/tmp/cache], + work_dir: %r[artifacts/tmp/work] context "object store is REMOTE" do before do @@ -22,7 +22,7 @@ RSpec.describe JobArtifactUploader do include_context 'with storage', described_class::Store::REMOTE it_behaves_like "builds correct paths", - store_dir: %r[\h{2}/\h{2}/\h{64}/\d{4}_\d{1,2}_\d{1,2}/\d+/\d+\z] + store_dir: %r[\h{2}/\h{2}/\h{64}/\d{4}_\d{1,2}_\d{1,2}/\d+/\d+\z] describe '#cdn_enabled_url' do it 'returns URL and false' do diff --git a/spec/uploaders/lfs_object_uploader_spec.rb b/spec/uploaders/lfs_object_uploader_spec.rb index b85892a42b5..9bbfd910ada 100644 --- a/spec/uploaders/lfs_object_uploader_spec.rb +++ b/spec/uploaders/lfs_object_uploader_spec.rb @@ -10,9 +10,9 @@ RSpec.describe LfsObjectUploader do subject { uploader } it_behaves_like "builds correct paths", - store_dir: %r[\h{2}/\h{2}], - cache_dir: %r[/lfs-objects/tmp/cache], - work_dir: %r[/lfs-objects/tmp/work] + store_dir: %r[\h{2}/\h{2}], + cache_dir: %r[/lfs-objects/tmp/cache], + work_dir: %r[/lfs-objects/tmp/work] context "object store is REMOTE" do before do @@ -22,7 +22,7 @@ RSpec.describe LfsObjectUploader do include_context 'with storage', described_class::Store::REMOTE it_behaves_like "builds correct paths", - store_dir: %r[\h{2}/\h{2}] + store_dir: %r[\h{2}/\h{2}] end describe 'remote file' do diff --git a/spec/uploaders/object_storage/cdn/google_cdn_spec.rb b/spec/uploaders/object_storage/cdn/google_cdn_spec.rb index 184c664f6dc..96413f622e8 100644 --- a/spec/uploaders/object_storage/cdn/google_cdn_spec.rb +++ b/spec/uploaders/object_storage/cdn/google_cdn_spec.rb @@ -99,9 +99,10 @@ RSpec.describe ObjectStorage::CDN::GoogleCDN, let(:path) { '/path/to/file.txt' } let(:expiration) { (Time.current + 10.minutes).utc.to_i } let(:cdn_query_params) { "Expires=#{expiration}&KeyName=#{key_name}" } + let(:encoded_path) { Addressable::URI.encode_component(path, Addressable::URI::CharacterClasses::PATH) } def verify_signature(url, unsigned_url) - expect(url).to start_with("#{options[:url]}#{path}") + expect(url).to start_with("#{options[:url]}#{encoded_path}") uri = Addressable::URI.parse(url) query = uri.query_values @@ -116,6 +117,16 @@ RSpec.describe ObjectStorage::CDN::GoogleCDN, end end + context 'with UTF-8 characters in path' do + let(:path) { "/path/to/©️job🧪" } + let(:url) { subject.signed_url(path) } + let(:unsigned_url) { "#{options[:url]}#{encoded_path}?#{cdn_query_params}" } + + it 'returns a valid signed URL' do + verify_signature(url, unsigned_url) + end + end + context 'with default query parameters' do let(:url) { subject.signed_url(path) } let(:unsigned_url) { "#{options[:url]}#{path}?#{cdn_query_params}" } diff --git a/spec/uploaders/object_storage/cdn_spec.rb b/spec/uploaders/object_storage/cdn_spec.rb index d6c638297fa..28b3313428b 100644 --- a/spec/uploaders/object_storage/cdn_spec.rb +++ b/spec/uploaders/object_storage/cdn_spec.rb @@ -3,19 +3,6 @@ require 'spec_helper' RSpec.describe ObjectStorage::CDN, feature_category: :build_artifacts do - let(:cdn_options) do - { - 'object_store' => { - 'cdn' => { - 'provider' => 'google', - 'url' => 'https://gitlab.example.com', - 'key_name' => 'test-key', - 'key' => Base64.urlsafe_encode64('12345') - } - } - }.freeze - end - let(:uploader_class) do Class.new(GitlabUploader) do include ObjectStorage::Concern @@ -39,44 +26,66 @@ RSpec.describe ObjectStorage::CDN, feature_category: :build_artifacts do subject { uploader_class.new(object, :file) } context 'with CDN config' do + let(:cdn_options) do + { + 'object_store' => { + 'cdn' => { + 'provider' => cdn_provider, + 'url' => 'https://gitlab.example.com', + 'key_name' => 'test-key', + 'key' => Base64.urlsafe_encode64('12345') + } + } + }.freeze + end + before do stub_artifacts_object_storage(enabled: true) - uploader_class.options = Settingslogic.new(Gitlab.config.uploads.deep_merge(cdn_options)) + options = Gitlab.config.uploads.deep_merge(cdn_options) + allow(uploader_class).to receive(:options).and_return(options) end - describe '#cdn_enabled_url' do - it 'calls #cdn_signed_url' do - expect(subject).not_to receive(:url) - expect(subject).to receive(:cdn_signed_url).with(query_params).and_call_original + context 'with a known CDN provider' do + let(:cdn_provider) { 'google' } - result = subject.cdn_enabled_url(public_ip, query_params) + describe '#cdn_enabled_url' do + it 'calls #cdn_signed_url' do + expect(subject).not_to receive(:url) + expect(subject).to receive(:cdn_signed_url).with(query_params).and_call_original + + result = subject.cdn_enabled_url(public_ip, query_params) - expect(result.used_cdn).to be true + expect(result.used_cdn).to be true + end end - end - describe '#use_cdn?' do - it 'returns true' do - expect(subject.use_cdn?(public_ip)).to be true + describe '#use_cdn?' do + it 'returns true' do + expect(subject.use_cdn?(public_ip)).to be true + end end - end - describe '#cdn_signed_url' do - it 'returns a URL' do - expect_next_instance_of(ObjectStorage::CDN::GoogleCDN) do |cdn| - expect(cdn).to receive(:signed_url).and_return("https://cdn.example.com/path") + describe '#cdn_signed_url' do + it 'returns a URL' do + expect_next_instance_of(ObjectStorage::CDN::GoogleCDN) do |cdn| + expect(cdn).to receive(:signed_url).and_return("https://cdn.example.com/path") + end + + expect(subject.cdn_signed_url).to eq("https://cdn.example.com/path") end + end + end + + context 'with an unknown CDN provider' do + let(:cdn_provider) { 'amazon' } - expect(subject.cdn_signed_url).to eq("https://cdn.example.com/path") + it 'raises an error' do + expect { subject.use_cdn?(public_ip) }.to raise_error("Unknown CDN provider: amazon") end end end context 'without CDN config' do - before do - uploader_class.options = Gitlab.config.uploads - end - describe '#cdn_enabled_url' do it 'calls #url' do expect(subject).not_to receive(:cdn_signed_url) @@ -94,15 +103,4 @@ RSpec.describe ObjectStorage::CDN, feature_category: :build_artifacts do end end end - - context 'with an unknown CDN provider' do - before do - cdn_options['object_store']['cdn']['provider'] = 'amazon' - uploader_class.options = Settingslogic.new(Gitlab.config.uploads.deep_merge(cdn_options)) - end - - it 'raises an error' do - expect { subject.use_cdn?(public_ip) }.to raise_error("Unknown CDN provider: amazon") - end - end end diff --git a/spec/uploaders/object_storage_spec.rb b/spec/uploaders/object_storage_spec.rb index 5344dbeb512..1566021934a 100644 --- a/spec/uploaders/object_storage_spec.rb +++ b/spec/uploaders/object_storage_spec.rb @@ -8,7 +8,7 @@ class Implementation < GitlabUploader include ::RecordsUploads::Concern prepend ::ObjectStorage::Extension::RecordsUploads - storage_options Gitlab.config.uploads + storage_location :uploads private @@ -18,10 +18,12 @@ class Implementation < GitlabUploader end end -RSpec.describe ObjectStorage do +# TODO: Update feature_category once object storage group ownership has been determined. +RSpec.describe ObjectStorage, :clean_gitlab_redis_shared_state, feature_category: :shared do let(:uploader_class) { Implementation } let(:object) { build_stubbed(:user) } - let(:uploader) { uploader_class.new(object, :file) } + let(:file_column) { :file } + let(:uploader) { uploader_class.new(object, file_column) } describe '#object_store=' do before do @@ -103,6 +105,34 @@ RSpec.describe ObjectStorage do expect(subject).to eq("my/prefix/user/#{object.id}/filename") end end + + context 'when model has final path defined for the file column' do + # Changing this to `foo` to make a point that not all uploaders are mounted + # as `file`. They can be mounted as different names, for example, `avatar`. + let(:file_column) { :foo } + + before do + allow(object).to receive(:foo_final_path).and_return('123-final-path') + end + + it 'uses the final path instead' do + expect(subject).to eq('123-final-path') + end + + context 'and a bucket prefix is configured' do + before do + allow(uploader_class).to receive(:object_store_options) do + double( + bucket_prefix: 'my/prefix' + ) + end + end + + it 'uses the prefix with the final path' do + expect(subject).to eq("my/prefix/123-final-path") + end + end + end end end end @@ -446,7 +476,7 @@ RSpec.describe ObjectStorage do end describe '#fog_credentials' do - let(:connection) { Settingslogic.new("provider" => "AWS") } + let(:connection) { GitlabSettings::Options.build("provider" => "AWS") } before do allow(uploader_class).to receive(:options) do @@ -479,7 +509,7 @@ RSpec.describe ObjectStorage do } end - let(:options) { Settingslogic.new(raw_options) } + let(:options) { GitlabSettings::Options.build(raw_options) } before do allow(uploader_class).to receive(:options) do @@ -494,8 +524,15 @@ RSpec.describe ObjectStorage do describe '.workhorse_authorize' do let(:has_length) { true } let(:maximum_size) { nil } + let(:use_final_store_path) { false } - subject { uploader_class.workhorse_authorize(has_length: has_length, maximum_size: maximum_size) } + subject do + uploader_class.workhorse_authorize( + has_length: has_length, + maximum_size: maximum_size, + use_final_store_path: use_final_store_path + ) + end context 'when FIPS is enabled', :fips_mode do it 'response enables FIPS' do @@ -528,18 +565,23 @@ RSpec.describe ObjectStorage do shared_examples 'uses remote storage' do it_behaves_like 'returns the maximum size given' do - it "returns remote store" do + it "returns remote object properties for a temporary upload" do is_expected.to have_key(:RemoteObject) expect(subject[:RemoteObject]).to have_key(:ID) expect(subject[:RemoteObject]).to include(Timeout: a_kind_of(Integer)) expect(subject[:RemoteObject][:Timeout]).to be(ObjectStorage::DirectUpload::TIMEOUT) - expect(subject[:RemoteObject]).to have_key(:GetURL) - expect(subject[:RemoteObject]).to have_key(:DeleteURL) - expect(subject[:RemoteObject]).to have_key(:StoreURL) - expect(subject[:RemoteObject][:GetURL]).to include(described_class::TMP_UPLOAD_PATH) - expect(subject[:RemoteObject][:DeleteURL]).to include(described_class::TMP_UPLOAD_PATH) - expect(subject[:RemoteObject][:StoreURL]).to include(described_class::TMP_UPLOAD_PATH) + + upload_path = File.join(described_class::TMP_UPLOAD_PATH, subject[:RemoteObject][:ID]) + + expect(subject[:RemoteObject][:GetURL]).to include(upload_path) + expect(subject[:RemoteObject][:DeleteURL]).to include(upload_path) + expect(subject[:RemoteObject][:StoreURL]).to include(upload_path) + expect(subject[:RemoteObject][:SkipDelete]).to eq(false) + + ::Gitlab::Redis::SharedState.with do |redis| + expect(redis.hlen(ObjectStorage::PendingDirectUpload::KEY)).to be_zero + end end end end @@ -551,12 +593,12 @@ RSpec.describe ObjectStorage do expect(subject[:RemoteObject]).to have_key(:MultipartUpload) expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:PartSize) - expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:PartURLs) - expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:CompleteURL) - expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:AbortURL) - expect(subject[:RemoteObject][:MultipartUpload][:PartURLs]).to all(include(described_class::TMP_UPLOAD_PATH)) - expect(subject[:RemoteObject][:MultipartUpload][:CompleteURL]).to include(described_class::TMP_UPLOAD_PATH) - expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to include(described_class::TMP_UPLOAD_PATH) + + upload_path = File.join(described_class::TMP_UPLOAD_PATH, subject[:RemoteObject][:ID]) + + expect(subject[:RemoteObject][:MultipartUpload][:PartURLs]).to all(include(upload_path)) + expect(subject[:RemoteObject][:MultipartUpload][:CompleteURL]).to include(upload_path) + expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to include(upload_path) end end end @@ -570,6 +612,80 @@ RSpec.describe ObjectStorage do end end + shared_examples 'handling object storage final upload path' do |multipart| + context 'when use_final_store_path is true' do + let(:use_final_store_path) { true } + let(:final_store_path) { File.join('@final', 'abc', '123', 'somefilename') } + let(:escaped_path) { escape_path(final_store_path) } + + before do + stub_object_storage_multipart_init_with_final_store_path("#{storage_url}#{final_store_path}") if multipart + + allow(uploader_class).to receive(:generate_final_store_path).and_return(final_store_path) + end + + it 'uses the full path instead of the temporary one' do + expect(subject[:RemoteObject][:ID]).to eq(final_store_path) + + expect(subject[:RemoteObject][:GetURL]).to include(escaped_path) + expect(subject[:RemoteObject][:StoreURL]).to include(escaped_path) + + if multipart + expect(subject[:RemoteObject][:MultipartUpload][:PartURLs]).to all(include(escaped_path)) + expect(subject[:RemoteObject][:MultipartUpload][:CompleteURL]).to include(escaped_path) + expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to include(escaped_path) + end + + expect(subject[:RemoteObject][:SkipDelete]).to eq(true) + + expect( + ObjectStorage::PendingDirectUpload.exists?(uploader_class.storage_location_identifier, final_store_path) + ).to eq(true) + end + + context 'and bucket prefix is configured' do + let(:prefixed_final_store_path) { "my/prefix/#{final_store_path}" } + let(:escaped_path) { escape_path(prefixed_final_store_path) } + + before do + allow(uploader_class.object_store_options).to receive(:bucket_prefix).and_return('my/prefix') + + if multipart + stub_object_storage_multipart_init_with_final_store_path("#{storage_url}#{prefixed_final_store_path}") + end + end + + it 'sets the remote object ID to the final path without prefix' do + expect(subject[:RemoteObject][:ID]).to eq(final_store_path) + end + + it 'returns the final path with prefix' do + expect(subject[:RemoteObject][:GetURL]).to include(escaped_path) + expect(subject[:RemoteObject][:StoreURL]).to include(escaped_path) + + if multipart + expect(subject[:RemoteObject][:MultipartUpload][:PartURLs]).to all(include(escaped_path)) + expect(subject[:RemoteObject][:MultipartUpload][:CompleteURL]).to include(escaped_path) + expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to include(escaped_path) + end + end + + it 'creates the pending upload entry without the prefix' do + is_expected.to have_key(:RemoteObject) + + expect( + ObjectStorage::PendingDirectUpload.exists?(uploader_class.storage_location_identifier, final_store_path) + ).to eq(true) + end + end + end + + def escape_path(path) + # This is what the private method Fog::AWS::Storage#object_to_path will do to the object name + Fog::AWS.escape(path).gsub('%2F', '/') + end + end + context 'when object storage is disabled' do before do allow(Gitlab.config.uploads.object_store).to receive(:enabled) { false } @@ -613,6 +729,8 @@ RSpec.describe ObjectStorage do expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) end end + + it_behaves_like 'handling object storage final upload path' end context 'for unknown length' do @@ -633,6 +751,8 @@ RSpec.describe ObjectStorage do expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to start_with(storage_url) end end + + it_behaves_like 'handling object storage final upload path', :multipart end end @@ -660,6 +780,8 @@ RSpec.describe ObjectStorage do expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) end end + + it_behaves_like 'handling object storage final upload path' end context 'for unknown length' do @@ -673,6 +795,8 @@ RSpec.describe ObjectStorage do expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) end end + + it_behaves_like 'handling object storage final upload path' end end @@ -701,6 +825,8 @@ RSpec.describe ObjectStorage do expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) end end + + it_behaves_like 'handling object storage final upload path' end context 'for unknown length' do @@ -721,6 +847,8 @@ RSpec.describe ObjectStorage do expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to start_with(storage_url) end end + + it_behaves_like 'handling object storage final upload path', :multipart end end end @@ -880,9 +1008,10 @@ RSpec.describe ObjectStorage do expect(uploader).to be_exists expect(uploader).to be_cached + expect(uploader.cache_only).to be_falsey expect(uploader).not_to be_file_storage expect(uploader.path).not_to be_nil - expect(uploader.path).not_to include('tmp/cache') + expect(uploader.path).to include('tmp/uploads') expect(uploader.path).not_to include('tmp/cache') expect(uploader.object_store).to eq(described_class::Store::REMOTE) end @@ -905,41 +1034,167 @@ RSpec.describe ObjectStorage do expect(uploader.object_store).to eq(described_class::Store::REMOTE) end end + + context 'when uploaded file remote_id matches a pending direct upload entry' do + let(:object) { build_stubbed(:ci_job_artifact) } + let(:final_path) { '@final/test/123123' } + let(:fog_config) { Gitlab.config.uploads.object_store } + let(:bucket) { 'uploads' } + let(:uploaded_file) { UploadedFile.new(temp_file.path, filename: "my_file.txt", remote_id: final_path) } + let(:fog_file_path) { final_path } + + let(:fog_connection_2) do + stub_object_storage_uploader( + config: fog_config, + uploader: uploader_class, + direct_upload: true + ) + end + + let!(:fog_file_2) do + fog_connection_2.directories.new(key: bucket).files.create( # rubocop:disable Rails/SaveBang + key: fog_file_path, + body: 'content' + ) + end + + before do + ObjectStorage::PendingDirectUpload.prepare( + uploader_class.storage_location_identifier, + final_path + ) + end + + it 'file to be cached and remote stored with final path set' do + expect { subject }.not_to raise_error + + expect(uploader).to be_exists + expect(uploader).to be_cached + expect(uploader.cache_only).to be_falsey + expect(uploader).not_to be_file_storage + expect(uploader.path).to eq(uploaded_file.remote_id) + expect(uploader.object_store).to eq(described_class::Store::REMOTE) + + expect(object.file_final_path).to eq(uploaded_file.remote_id) + end + + context 'when bucket prefix is configured' do + let(:fog_config) do + Gitlab.config.uploads.object_store.tap do |config| + config[:remote_directory] = 'main-bucket' + config[:bucket_prefix] = 'uploads' + end + end + + let(:bucket) { 'main-bucket' } + let(:fog_file_path) { "uploads/#{final_path}" } + + it 'stores the file final path in the db without the prefix' do + expect { subject }.not_to raise_error + + expect(uploader.store_path).to eq("uploads/#{final_path}") + expect(object.file_final_path).to eq(final_path) + end + end + + context 'when file is stored' do + subject do + uploader.store!(uploaded_file) + end + + it 'file to be remotely stored in permament location' do + subject + + expect(uploader).to be_exists + expect(uploader).not_to be_cached + expect(uploader.path).to eq(uploaded_file.remote_id) + end + + it 'does not trigger Carrierwave copy and delete because it is already in the final location' do + expect_next_instance_of(CarrierWave::Storage::Fog::File) do |instance| + expect(instance).not_to receive(:copy_to) + expect(instance).not_to receive(:delete) + end + + subject + end + end + end end end end end describe '#retrieve_from_store!' do - [:group, :project, :user].each do |model| - context "for #{model}s" do - let(:models) { create_list(model, 3, :with_avatar).map(&:reload) } - let(:avatars) { models.map(&:avatar) } + context 'uploaders that includes the RecordsUploads extension' do + [:group, :project, :user].each do |model| + context "for #{model}s" do + let(:models) { create_list(model, 3, :with_avatar).map(&:reload) } + let(:avatars) { models.map(&:avatar) } + + it 'batches fetching uploads from the database' do + # Ensure that these are all created and fully loaded before we start + # running queries for avatars + models + + expect { avatars }.not_to exceed_query_limit(1) + end + + it 'does not attempt to replace methods' do + models.each do |model| + expect(model.avatar.upload).to receive(:method_missing).and_call_original - it 'batches fetching uploads from the database' do - # Ensure that these are all created and fully loaded before we start - # running queries for avatars - models + model.avatar.upload.path + end + end - expect { avatars }.not_to exceed_query_limit(1) + it 'fetches a unique upload for each model' do + expect(avatars.map(&:url).uniq).to eq(avatars.map(&:url)) + expect(avatars.map(&:upload).uniq).to eq(avatars.map(&:upload)) + end end + end + end - it 'does not attempt to replace methods' do - models.each do |model| - expect(model.avatar.upload).to receive(:method_missing).and_call_original + describe 'filename' do + let(:model) { create(:ci_job_artifact, :remote_store, :archive) } - model.avatar.upload.path - end + before do + stub_artifacts_object_storage + end + + shared_examples 'ensuring correct filename' do + it 'uses the original filename' do + expect(model.reload.file.filename).to eq('ci_build_artifacts.zip') end + end - it 'fetches a unique upload for each model' do - expect(avatars.map(&:url).uniq).to eq(avatars.map(&:url)) - expect(avatars.map(&:upload).uniq).to eq(avatars.map(&:upload)) + context 'when model has final path defined for the file column' do + before do + model.update_column(:file_final_path, 'some/final/path/abc-123') end + + it_behaves_like 'ensuring correct filename' + end + + context 'when model has no final path defined for the file column' do + it_behaves_like 'ensuring correct filename' end end end + describe '.generate_final_store_path' do + subject(:final_path) { uploader_class.generate_final_store_path } + + before do + allow(Digest::SHA2).to receive(:hexdigest).and_return('somehash1234') + end + + it 'returns the generated hashed path' do + expect(final_path).to eq('@final/so/me/hash1234') + end + end + describe 'OpenFile' do subject { ObjectStorage::Concern::OpenFile.new(file) } diff --git a/spec/uploaders/packages/composer/cache_uploader_spec.rb b/spec/uploaders/packages/composer/cache_uploader_spec.rb index 7ceaa24f463..7eea4a839ab 100644 --- a/spec/uploaders/packages/composer/cache_uploader_spec.rb +++ b/spec/uploaders/packages/composer/cache_uploader_spec.rb @@ -9,9 +9,9 @@ RSpec.describe Packages::Composer::CacheUploader do subject { uploader } it_behaves_like "builds correct paths", - store_dir: %r[^\h{2}/\h{2}/\h{64}/packages/composer_cache/\d+$], - cache_dir: %r[/packages/tmp/cache], - work_dir: %r[/packages/tmp/work] + store_dir: %r[^\h{2}/\h{2}/\h{64}/packages/composer_cache/\d+$], + cache_dir: %r[/packages/tmp/cache], + work_dir: %r[/packages/tmp/work] context 'object store is remote' do before do @@ -21,7 +21,7 @@ RSpec.describe Packages::Composer::CacheUploader do include_context 'with storage', described_class::Store::REMOTE it_behaves_like "builds correct paths", - store_dir: %r[^\h{2}/\h{2}/\h{64}/packages/composer_cache/\d+$] + store_dir: %r[^\h{2}/\h{2}/\h{64}/packages/composer_cache/\d+$] end describe 'remote file' do diff --git a/spec/uploaders/packages/debian/component_file_uploader_spec.rb b/spec/uploaders/packages/debian/component_file_uploader_spec.rb index bee82fb2715..84ba751c737 100644 --- a/spec/uploaders/packages/debian/component_file_uploader_spec.rb +++ b/spec/uploaders/packages/debian/component_file_uploader_spec.rb @@ -12,9 +12,9 @@ RSpec.describe Packages::Debian::ComponentFileUploader do subject { uploader } it_behaves_like "builds correct paths", - store_dir: %r[^\h{2}/\h{2}/\h{64}/debian_#{container_type}_component_file/\d+$], - cache_dir: %r[/packages/tmp/cache$], - work_dir: %r[/packages/tmp/work$] + store_dir: %r[^\h{2}/\h{2}/\h{64}/debian_#{container_type}_component_file/\d+$], + cache_dir: %r[/packages/tmp/cache$], + work_dir: %r[/packages/tmp/work$] context 'object store is remote' do before do @@ -24,9 +24,9 @@ RSpec.describe Packages::Debian::ComponentFileUploader do include_context 'with storage', described_class::Store::REMOTE it_behaves_like "builds correct paths", - store_dir: %r[^\h{2}/\h{2}/\h{64}/debian_#{container_type}_component_file/\d+$], - cache_dir: %r[/packages/tmp/cache$], - work_dir: %r[/packages/tmp/work$] + store_dir: %r[^\h{2}/\h{2}/\h{64}/debian_#{container_type}_component_file/\d+$], + cache_dir: %r[/packages/tmp/cache$], + work_dir: %r[/packages/tmp/work$] end describe 'remote file' do diff --git a/spec/uploaders/packages/debian/distribution_release_file_uploader_spec.rb b/spec/uploaders/packages/debian/distribution_release_file_uploader_spec.rb index 96655edb186..df630569856 100644 --- a/spec/uploaders/packages/debian/distribution_release_file_uploader_spec.rb +++ b/spec/uploaders/packages/debian/distribution_release_file_uploader_spec.rb @@ -12,9 +12,9 @@ RSpec.describe Packages::Debian::DistributionReleaseFileUploader do subject { uploader } it_behaves_like "builds correct paths", - store_dir: %r[^\h{2}/\h{2}/\h{64}/debian_#{container_type}_distribution/\d+$], - cache_dir: %r[/packages/tmp/cache$], - work_dir: %r[/packages/tmp/work$] + store_dir: %r[^\h{2}/\h{2}/\h{64}/debian_#{container_type}_distribution/\d+$], + cache_dir: %r[/packages/tmp/cache$], + work_dir: %r[/packages/tmp/work$] context 'object store is remote' do before do @@ -24,9 +24,9 @@ RSpec.describe Packages::Debian::DistributionReleaseFileUploader do include_context 'with storage', described_class::Store::REMOTE it_behaves_like "builds correct paths", - store_dir: %r[^\h{2}/\h{2}/\h{64}/debian_#{container_type}_distribution/\d+$], - cache_dir: %r[/packages/tmp/cache$], - work_dir: %r[/packages/tmp/work$] + store_dir: %r[^\h{2}/\h{2}/\h{64}/debian_#{container_type}_distribution/\d+$], + cache_dir: %r[/packages/tmp/cache$], + work_dir: %r[/packages/tmp/work$] end describe 'remote file' do diff --git a/spec/uploaders/packages/npm/metadata_cache_uploader_spec.rb b/spec/uploaders/packages/npm/metadata_cache_uploader_spec.rb new file mode 100644 index 00000000000..0bcf05932a5 --- /dev/null +++ b/spec/uploaders/packages/npm/metadata_cache_uploader_spec.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Packages::Npm::MetadataCacheUploader, feature_category: :package_registry do + let(:object_storage_key) { 'object/storage/key' } + let(:npm_metadata_cache) { build_stubbed(:npm_metadata_cache, object_storage_key: object_storage_key) } + + subject { described_class.new(npm_metadata_cache, :file) } + + describe '#filename' do + it 'returns metadata.json' do + expect(subject.filename).to eq('metadata.json') + end + end + + describe '#store_dir' do + it 'uses the object_storage_key' do + expect(subject.store_dir).to eq(object_storage_key) + end + + context 'without the object_storage_key' do + let(:object_storage_key) { nil } + + it 'raises the error' do + expect { subject.store_dir } + .to raise_error( + described_class::ObjectNotReadyError, + 'Packages::Npm::MetadataCache model not ready' + ) + end + end + end +end diff --git a/spec/uploaders/packages/package_file_uploader_spec.rb b/spec/uploaders/packages/package_file_uploader_spec.rb index 7d270ad03c9..ddd9823d55c 100644 --- a/spec/uploaders/packages/package_file_uploader_spec.rb +++ b/spec/uploaders/packages/package_file_uploader_spec.rb @@ -9,9 +9,9 @@ RSpec.describe Packages::PackageFileUploader do subject { uploader } it_behaves_like "builds correct paths", - store_dir: %r[^\h{2}/\h{2}/\h{64}/packages/\d+/files/\d+$], - cache_dir: %r[/packages/tmp/cache], - work_dir: %r[/packages/tmp/work] + store_dir: %r[^\h{2}/\h{2}/\h{64}/packages/\d+/files/\d+$], + cache_dir: %r[/packages/tmp/cache], + work_dir: %r[/packages/tmp/work] context 'object store is remote' do before do @@ -21,7 +21,7 @@ RSpec.describe Packages::PackageFileUploader do include_context 'with storage', described_class::Store::REMOTE it_behaves_like "builds correct paths", - store_dir: %r[^\h{2}/\h{2}/\h{64}/packages/\d+/files/\d+$] + store_dir: %r[^\h{2}/\h{2}/\h{64}/packages/\d+/files/\d+$] end describe 'remote file' do diff --git a/spec/uploaders/packages/rpm/repository_file_uploader_spec.rb b/spec/uploaders/packages/rpm/repository_file_uploader_spec.rb index b3767ae179a..a36a035fde3 100644 --- a/spec/uploaders/packages/rpm/repository_file_uploader_spec.rb +++ b/spec/uploaders/packages/rpm/repository_file_uploader_spec.rb @@ -9,9 +9,9 @@ RSpec.describe Packages::Rpm::RepositoryFileUploader do subject { uploader } it_behaves_like 'builds correct paths', - store_dir: %r[^\h{2}/\h{2}/\h{64}/projects/\d+/rpm/repository_files/\d+$], - cache_dir: %r{/packages/tmp/cache}, - work_dir: %r{/packages/tmp/work} + store_dir: %r[^\h{2}/\h{2}/\h{64}/projects/\d+/rpm/repository_files/\d+$], + cache_dir: %r{/packages/tmp/cache}, + work_dir: %r{/packages/tmp/work} context 'when object store is remote' do before do @@ -21,7 +21,7 @@ RSpec.describe Packages::Rpm::RepositoryFileUploader do include_context 'with storage', described_class::Store::REMOTE it_behaves_like 'builds correct paths', - store_dir: %r[^\h{2}/\h{2}/\h{64}/projects/\d+/rpm/repository_files/\d+$] + store_dir: %r[^\h{2}/\h{2}/\h{64}/projects/\d+/rpm/repository_files/\d+$] end describe 'remote file' do diff --git a/spec/uploaders/pages/deployment_uploader_spec.rb b/spec/uploaders/pages/deployment_uploader_spec.rb index 1832f73bd67..7686efd4fe4 100644 --- a/spec/uploaders/pages/deployment_uploader_spec.rb +++ b/spec/uploaders/pages/deployment_uploader_spec.rb @@ -13,9 +13,9 @@ RSpec.describe Pages::DeploymentUploader do subject { uploader } it_behaves_like "builds correct paths", - store_dir: %r[/\h{2}/\h{2}/\h{64}/pages_deployments/\d+], - cache_dir: %r[pages/@hashed/tmp/cache], - work_dir: %r[pages/@hashed/tmp/work] + store_dir: %r[/\h{2}/\h{2}/\h{64}/pages_deployments/\d+], + cache_dir: %r[pages/@hashed/tmp/cache], + work_dir: %r[pages/@hashed/tmp/work] context 'when object store is REMOTE' do before do diff --git a/spec/uploaders/personal_file_uploader_spec.rb b/spec/uploaders/personal_file_uploader_spec.rb index 1373ccac23d..58edf3f093d 100644 --- a/spec/uploaders/personal_file_uploader_spec.rb +++ b/spec/uploaders/personal_file_uploader_spec.rb @@ -50,9 +50,9 @@ RSpec.describe PersonalFileUploader do context 'object_store is LOCAL' do it_behaves_like 'builds correct paths', - store_dir: %r[uploads/-/system/personal_snippet/\d+/\h+], - upload_path: %r[\h+/\S+], - absolute_path: %r[#{CarrierWave.root}/uploads/-/system/personal_snippet/\d+/\h+/\S+$] + store_dir: %r[uploads/-/system/personal_snippet/\d+/\h+], + upload_path: %r[\h+/\S+], + absolute_path: %r[#{CarrierWave.root}/uploads/-/system/personal_snippet/\d+/\h+/\S+$] it_behaves_like '#base_dir' it_behaves_like '#to_h' @@ -66,8 +66,8 @@ RSpec.describe PersonalFileUploader do include_context 'with storage', described_class::Store::REMOTE it_behaves_like 'builds correct paths', - store_dir: %r[\d+/\h+], - upload_path: %r[^personal_snippet/\d+/\h+/<filename>] + store_dir: %r[\d+/\h+], + upload_path: %r[^personal_snippet/\d+/\h+/<filename>] it_behaves_like '#base_dir' it_behaves_like '#to_h' |