diff options
Diffstat (limited to 'spec/services')
5 files changed, 421 insertions, 1 deletions
diff --git a/spec/services/projects/import_service_spec.rb b/spec/services/projects/import_service_spec.rb index 30c89ebd821..b3815045792 100644 --- a/spec/services/projects/import_service_spec.rb +++ b/spec/services/projects/import_service_spec.rb @@ -3,9 +3,17 @@ require 'spec_helper' describe Projects::ImportService do let!(:project) { create(:project) } let(:user) { project.creator } + let(:import_url) { 'http://www.gitlab.com/demo/repo.git' } + let(:oid_download_links) { { 'oid1' => "#{import_url}/gitlab-lfs/objects/oid1", 'oid2' => "#{import_url}/gitlab-lfs/objects/oid2" } } subject { described_class.new(project, user) } + before do + allow(project).to receive(:lfs_enabled?).and_return(true) + allow_any_instance_of(Projects::LfsPointers::LfsDownloadService).to receive(:execute) + allow_any_instance_of(Projects::LfsPointers::LfsImportService).to receive(:execute).and_return(oid_download_links) + end + describe '#async?' do it 'returns true for an asynchronous importer' do importer_class = double(:importer, async?: true) @@ -63,6 +71,15 @@ describe Projects::ImportService do expect(result[:status]).to eq :error expect(result[:message]).to eq "Error importing repository #{project.import_url} into #{project.full_path} - The repository could not be created." end + + context 'when repository creation succeeds' do + it 'does not download lfs files' do + expect_any_instance_of(Projects::LfsPointers::LfsImportService).not_to receive(:execute) + expect_any_instance_of(Projects::LfsPointers::LfsDownloadService).not_to receive(:execute) + + subject.execute + end + end end context 'with known url' do @@ -91,6 +108,15 @@ describe Projects::ImportService do expect(result[:status]).to eq :error end + + context 'when repository import scheduled' do + it 'does not download lfs objects' do + expect_any_instance_of(Projects::LfsPointers::LfsImportService).not_to receive(:execute) + expect_any_instance_of(Projects::LfsPointers::LfsDownloadService).not_to receive(:execute) + + subject.execute + end + end end context 'with a non Github repository' do @@ -99,9 +125,10 @@ describe Projects::ImportService do project.import_type = 'bitbucket' end - it 'succeeds if repository import is successfully' do + it 'succeeds if repository import is successfull' do expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).and_return(true) expect_any_instance_of(Gitlab::BitbucketImport::Importer).to receive(:execute).and_return(true) + expect_any_instance_of(Projects::LfsPointers::LfsImportService).to receive(:execute).and_return({}) result = subject.execute @@ -116,6 +143,29 @@ describe Projects::ImportService do expect(result[:status]).to eq :error expect(result[:message]).to eq "Error importing repository #{project.import_url} into #{project.full_path} - Failed to import the repository" end + + context 'when repository import scheduled' do + before do + allow_any_instance_of(Gitlab::Shell).to receive(:import_repository).and_return(true) + allow(subject).to receive(:import_data) + end + + it 'downloads lfs objects if lfs_enabled is enabled for project' do + allow(project).to receive(:lfs_enabled?).and_return(true) + expect_any_instance_of(Projects::LfsPointers::LfsImportService).to receive(:execute).and_return(oid_download_links) + expect_any_instance_of(Projects::LfsPointers::LfsDownloadService).to receive(:execute).twice + + subject.execute + end + + it 'does not download lfs objects if lfs_enabled is not enabled for project' do + allow(project).to receive(:lfs_enabled?).and_return(false) + expect_any_instance_of(Projects::LfsPointers::LfsImportService).not_to receive(:execute) + expect_any_instance_of(Projects::LfsPointers::LfsDownloadService).not_to receive(:execute) + + subject.execute + end + end end end @@ -147,6 +197,26 @@ describe Projects::ImportService do expect(result[:status]).to eq :error end + + context 'when importer' do + it 'has a custom repository importer it does not download lfs objects' do + allow(Gitlab::GithubImport::ParallelImporter).to receive(:imports_repository?).and_return(true) + + expect_any_instance_of(Projects::LfsPointers::LfsImportService).not_to receive(:execute) + expect_any_instance_of(Projects::LfsPointers::LfsDownloadService).not_to receive(:execute) + + subject.execute + end + + it 'does not have a custom repository importer downloads lfs objects' do + allow(Gitlab::GithubImport::ParallelImporter).to receive(:imports_repository?).and_return(false) + + expect_any_instance_of(Projects::LfsPointers::LfsImportService).to receive(:execute).and_return(oid_download_links) + expect_any_instance_of(Projects::LfsPointers::LfsDownloadService).to receive(:execute) + + subject.execute + end + end end context 'with blocked import_URL' do diff --git a/spec/services/projects/lfs_pointers/lfs_download_link_list_service_spec.rb b/spec/services/projects/lfs_pointers/lfs_download_link_list_service_spec.rb new file mode 100644 index 00000000000..d7a2829d5f8 --- /dev/null +++ b/spec/services/projects/lfs_pointers/lfs_download_link_list_service_spec.rb @@ -0,0 +1,102 @@ +require 'spec_helper' + +describe Projects::LfsPointers::LfsDownloadLinkListService do + let(:import_url) { 'http://www.gitlab.com/demo/repo.git' } + let(:lfs_endpoint) { "#{import_url}/info/lfs/objects/batch" } + let!(:project) { create(:project, import_url: import_url) } + let(:new_oids) { { 'oid1' => 123, 'oid2' => 125 } } + let(:remote_uri) { URI.parse(lfs_endpoint) } + + let(:objects_response) do + body = new_oids.map do |oid, size| + { + 'oid' => oid, + 'size' => size, + 'actions' => { + 'download' => { 'href' => "#{import_url}/gitlab-lfs/objects/#{oid}" } + } + } + end + + Struct.new(:success?, :objects).new(true, body) + end + + let(:invalid_object_response) do + [ + 'oid' => 'whatever', + 'size' => 123 + ] + end + + subject { described_class.new(project, remote_uri: remote_uri) } + + before do + allow(project).to receive(:lfs_enabled?).and_return(true) + allow(Gitlab::HTTP).to receive(:post).and_return(objects_response) + end + + describe '#execute' do + it 'retrieves each download link of every non existent lfs object' do + subject.execute(new_oids).each do |oid, link| + expect(link).to eq "#{import_url}/gitlab-lfs/objects/#{oid}" + end + end + + context 'credentials' do + context 'when the download link and the lfs_endpoint have the same host' do + context 'when lfs_endpoint has credentials' do + let(:import_url) { 'http://user:password@www.gitlab.com/demo/repo.git' } + + it 'adds credentials to the download_link' do + result = subject.execute(new_oids) + + result.each do |oid, link| + expect(link.starts_with?('http://user:password@')).to be_truthy + end + end + end + + context 'when lfs_endpoint does not have any credentials' do + it 'does not add any credentials' do + result = subject.execute(new_oids) + + result.each do |oid, link| + expect(link.starts_with?('http://user:password@')).to be_falsey + end + end + end + end + + context 'when the download link and the lfs_endpoint have different hosts' do + let(:import_url_with_credentials) { 'http://user:password@www.otherdomain.com/demo/repo.git' } + let(:lfs_endpoint) { "#{import_url_with_credentials}/info/lfs/objects/batch" } + + it 'downloads without any credentials' do + result = subject.execute(new_oids) + + result.each do |oid, link| + expect(link.starts_with?('http://user:password@')).to be_falsey + end + end + end + end + end + + describe '#get_download_links' do + it 'raise errorif request fails' do + allow(Gitlab::HTTP).to receive(:post).and_return(Struct.new(:success?, :message).new(false, 'Failed request')) + + expect { subject.send(:get_download_links, new_oids) }.to raise_error(described_class::DownloadLinksError) + end + end + + describe '#parse_response_links' do + it 'does not add oid entry if href not found' do + expect(Rails.logger).to receive(:error).with("Link for Lfs Object with oid whatever not found or invalid.") + + result = subject.send(:parse_response_links, invalid_object_response) + + expect(result).to be_empty + end + end +end diff --git a/spec/services/projects/lfs_pointers/lfs_download_service_spec.rb b/spec/services/projects/lfs_pointers/lfs_download_service_spec.rb new file mode 100644 index 00000000000..6af5bfc7689 --- /dev/null +++ b/spec/services/projects/lfs_pointers/lfs_download_service_spec.rb @@ -0,0 +1,69 @@ +require 'spec_helper' + +describe Projects::LfsPointers::LfsDownloadService do + let(:project) { create(:project) } + let(:oid) { '9e548e25631dd9ce6b43afd6359ab76da2819d6a5b474e66118c7819e1d8b3e8' } + let(:download_link) { "http://gitlab.com/#{oid}" } + let(:lfs_content) do + <<~HEREDOC + whatever + HEREDOC + end + + subject { described_class.new(project) } + + before do + allow(project).to receive(:lfs_enabled?).and_return(true) + WebMock.stub_request(:get, download_link).to_return(body: lfs_content) + end + + describe '#execute' do + context 'when file download succeeds' do + it 'a new lfs object is created' do + expect { subject.execute(oid, download_link) }.to change { LfsObject.count }.from(0).to(1) + end + + it 'has the same oid' do + subject.execute(oid, download_link) + + expect(LfsObject.first.oid).to eq oid + end + + it 'stores the content' do + subject.execute(oid, download_link) + + expect(File.read(LfsObject.first.file.file.file)).to eq lfs_content + end + end + + context 'when file download fails' do + it 'no lfs object is created' do + expect { subject.execute(oid, download_link) }.to change { LfsObject.count } + end + end + + context 'when credentials present' do + let(:download_link_with_credentials) { "http://user:password@gitlab.com/#{oid}" } + + before do + WebMock.stub_request(:get, download_link).with(headers: { 'Authorization' => 'Basic dXNlcjpwYXNzd29yZA==' }).to_return(body: lfs_content) + end + + it 'the request adds authorization headers' do + subject.execute(oid, download_link_with_credentials) + end + end + + context 'when an lfs object with the same oid already exists' do + before do + create(:lfs_object, oid: 'oid') + end + + it 'does not download the file' do + expect(subject).not_to receive(:download_and_save_file) + + subject.execute('oid', download_link) + end + end + end +end diff --git a/spec/services/projects/lfs_pointers/lfs_import_service_spec.rb b/spec/services/projects/lfs_pointers/lfs_import_service_spec.rb new file mode 100644 index 00000000000..5a75fb38dec --- /dev/null +++ b/spec/services/projects/lfs_pointers/lfs_import_service_spec.rb @@ -0,0 +1,146 @@ +require 'spec_helper' + +describe Projects::LfsPointers::LfsImportService do + let(:import_url) { 'http://www.gitlab.com/demo/repo.git' } + let(:default_endpoint) { "#{import_url}/info/lfs/objects/batch"} + let(:group) { create(:group, lfs_enabled: true)} + let!(:project) { create(:project, namespace: group, import_url: import_url, lfs_enabled: true) } + let!(:lfs_objects_project) { create_list(:lfs_objects_project, 2, project: project) } + let!(:existing_lfs_objects) { LfsObject.pluck(:oid, :size).to_h } + let(:oids) { { 'oid1' => 123, 'oid2' => 125 } } + let(:oid_download_links) { { 'oid1' => "#{import_url}/gitlab-lfs/objects/oid1", 'oid2' => "#{import_url}/gitlab-lfs/objects/oid2" } } + let(:all_oids) { existing_lfs_objects.merge(oids) } + let(:remote_uri) { URI.parse(lfs_endpoint) } + + subject { described_class.new(project) } + + before do + allow(project.repository).to receive(:lfsconfig_for).and_return(nil) + allow(Gitlab.config.lfs).to receive(:enabled).and_return(true) + allow_any_instance_of(Projects::LfsPointers::LfsListService).to receive(:execute).and_return(all_oids) + end + + describe '#execute' do + context 'when no lfs pointer is linked' do + before do + allow_any_instance_of(Projects::LfsPointers::LfsLinkService).to receive(:execute).and_return([]) + allow_any_instance_of(Projects::LfsPointers::LfsDownloadLinkListService).to receive(:execute).and_return(oid_download_links) + expect(Projects::LfsPointers::LfsDownloadLinkListService).to receive(:new).with(project, remote_uri: URI.parse(default_endpoint)).and_call_original + end + + it 'retrieves all lfs pointers in the project repository' do + expect_any_instance_of(Projects::LfsPointers::LfsListService).to receive(:execute) + + subject.execute + end + + it 'links existent lfs objects to the project' do + expect_any_instance_of(Projects::LfsPointers::LfsLinkService).to receive(:execute) + + subject.execute + end + + it 'retrieves the download links of non existent objects' do + expect_any_instance_of(Projects::LfsPointers::LfsDownloadLinkListService).to receive(:execute).with(all_oids) + + subject.execute + end + end + + context 'when some lfs objects are linked' do + before do + allow_any_instance_of(Projects::LfsPointers::LfsLinkService).to receive(:execute).and_return(existing_lfs_objects.keys) + allow_any_instance_of(Projects::LfsPointers::LfsDownloadLinkListService).to receive(:execute).and_return(oid_download_links) + end + + it 'retrieves the download links of non existent objects' do + expect_any_instance_of(Projects::LfsPointers::LfsDownloadLinkListService).to receive(:execute).with(oids) + + subject.execute + end + end + + context 'when all lfs objects are linked' do + before do + allow_any_instance_of(Projects::LfsPointers::LfsLinkService).to receive(:execute).and_return(all_oids.keys) + allow_any_instance_of(Projects::LfsPointers::LfsDownloadLinkListService).to receive(:execute) + end + + it 'retrieves no download links' do + expect_any_instance_of(Projects::LfsPointers::LfsDownloadLinkListService).to receive(:execute).with({}).and_call_original + + expect(subject.execute).to be_empty + end + end + + context 'when lfsconfig file exists' do + before do + allow(project.repository).to receive(:lfsconfig_for).and_return("[lfs]\n\turl = #{lfs_endpoint}\n") + end + + context 'when url points to the same import url host' do + let(:lfs_endpoint) { "#{import_url}/different_endpoint" } + let(:service) { double } + + before do + allow(service).to receive(:execute) + end + it 'downloads lfs object using the new endpoint' do + expect(Projects::LfsPointers::LfsDownloadLinkListService).to receive(:new).with(project, remote_uri: remote_uri).and_return(service) + + subject.execute + end + + context 'when import url has credentials' do + let(:import_url) { 'http://user:password@www.gitlab.com/demo/repo.git'} + + it 'adds the credentials to the new endpoint' do + expect(Projects::LfsPointers::LfsDownloadLinkListService) + .to receive(:new).with(project, remote_uri: URI.parse("http://user:password@www.gitlab.com/demo/repo.git/different_endpoint")) + .and_return(service) + + subject.execute + end + + context 'when url has its own credentials' do + let(:lfs_endpoint) { "http://user1:password1@www.gitlab.com/demo/repo.git/different_endpoint" } + + it 'does not add the import url credentials' do + expect(Projects::LfsPointers::LfsDownloadLinkListService) + .to receive(:new).with(project, remote_uri: remote_uri) + .and_return(service) + + subject.execute + end + end + end + end + + context 'when url points to a third party service' do + let(:lfs_endpoint) { 'http://third_party_service.com/info/lfs/objects/' } + + it 'disables lfs from the project' do + expect(project.lfs_enabled?).to be_truthy + + subject.execute + + expect(project.lfs_enabled?).to be_falsey + end + + it 'does not download anything' do + expect_any_instance_of(Projects::LfsPointers::LfsListService).not_to receive(:execute) + + subject.execute + end + end + end + end + + describe '#default_endpoint_uri' do + let(:import_url) { 'http://www.gitlab.com/demo/repo' } + + it 'adds suffix .git if the url does not have it' do + expect(subject.send(:default_endpoint_uri).path).to match(/repo.git/) + end + end +end diff --git a/spec/services/projects/lfs_pointers/lfs_link_service_spec.rb b/spec/services/projects/lfs_pointers/lfs_link_service_spec.rb new file mode 100644 index 00000000000..b7b153655db --- /dev/null +++ b/spec/services/projects/lfs_pointers/lfs_link_service_spec.rb @@ -0,0 +1,33 @@ +require 'spec_helper' + +describe Projects::LfsPointers::LfsLinkService do + let!(:project) { create(:project, lfs_enabled: true) } + let!(:lfs_objects_project) { create_list(:lfs_objects_project, 2, project: project) } + let(:new_oids) { { 'oid1' => 123, 'oid2' => 125 } } + let(:all_oids) { LfsObject.pluck(:oid, :size).to_h.merge(new_oids) } + let(:new_lfs_object) { create(:lfs_object) } + let(:new_oid_list) { all_oids.merge(new_lfs_object.oid => new_lfs_object.size) } + + subject { described_class.new(project) } + + before do + allow(project).to receive(:lfs_enabled?).and_return(true) + end + + describe '#execute' do + it 'links existing lfs objects to the project' do + expect(project.all_lfs_objects.count).to eq 2 + + linked = subject.execute(new_oid_list.keys) + + expect(project.all_lfs_objects.count).to eq 3 + expect(linked.size).to eq 3 + end + + it 'returns linked oids' do + linked = lfs_objects_project.map(&:lfs_object).map(&:oid) << new_lfs_object.oid + + expect(subject.execute(new_oid_list.keys)).to eq linked + end + end +end |