summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--app/controllers/projects/jobs_controller.rb32
-rw-r--r--app/models/lfs_object.rb2
-rw-r--r--app/uploaders/job_artifact_uploader.rb1
-rw-r--r--db/schema.rb2
-rw-r--r--spec/ee/spec/finders/geo/attachment_registry_finder_spec.rb270
-rw-r--r--spec/ee/spec/lib/gitlab/geo/file_transfer_spec.rb22
-rw-r--r--spec/ee/spec/lib/gitlab/geo/log_cursor/daemon_spec.rb414
-rw-r--r--spec/ee/spec/services/geo/file_download_service_spec.rb227
-rw-r--r--spec/ee/spec/services/geo/files_expire_service_spec.rb51
-rw-r--r--spec/ee/spec/services/geo/hashed_storage_attachments_migration_service_spec.rb83
-rw-r--r--spec/ee/spec/workers/geo/file_download_dispatch_worker_spec.rb291
-rw-r--r--spec/factories/geo/event_log.rb121
-rw-r--r--spec/factories/uploads.rb1
-rw-r--r--spec/models/ci/job_artifact_spec.rb2
-rw-r--r--spec/support/shared_examples/uploaders/gitlab_uploader_shared_examples.rb48
-rw-r--r--spec/workers/object_storage_upload_worker_spec.rb (renamed from spec/ee/workers/object_storage_upload_worker_spec.rb)2
16 files changed, 76 insertions, 1493 deletions
diff --git a/app/controllers/projects/jobs_controller.rb b/app/controllers/projects/jobs_controller.rb
index 8041bdbb0bd..abf4be6250b 100644
--- a/app/controllers/projects/jobs_controller.rb
+++ b/app/controllers/projects/jobs_controller.rb
@@ -1,6 +1,4 @@
class Projects::JobsController < Projects::ApplicationController
- prepend EE::Projects::JobsController
-
before_action :build, except: [:index, :cancel_all]
before_action :authorize_read_build!,
@@ -119,11 +117,17 @@ class Projects::JobsController < Projects::ApplicationController
end
def raw
- build.trace.read do |stream|
- if stream.file?
- send_file stream.path, type: 'text/plain; charset=utf-8', disposition: 'inline'
- else
- render_404
+ if trace_artifact_file
+ send_upload(trace_artifact_file,
+ send_params: raw_send_params,
+ redirect_params: raw_redirect_params)
+ else
+ build.trace.read do |stream|
+ if stream.file?
+ send_file stream.path, type: 'text/plain; charset=utf-8', disposition: 'inline'
+ else
+ render_404
+ end
end
end
end
@@ -138,9 +142,21 @@ class Projects::JobsController < Projects::ApplicationController
return access_denied! unless can?(current_user, :erase_build, build)
end
+ def raw_send_params
+ { type: 'text/plain; charset=utf-8', disposition: 'inline' }
+ end
+
+ def raw_redirect_params
+ { query: { 'response-content-type' => 'text/plain; charset=utf-8', 'response-content-disposition' => 'inline' } }
+ end
+
+ def trace_artifact_file
+ @trace_artifact_file ||= build.job_artifacts_trace&.file
+ end
+
def build
@build ||= project.builds.find(params[:id])
- .present(current_user: current_user)
+ .present(current_user: current_user)
end
def build_path(build)
diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb
index 04c75d827e0..54840fb084d 100644
--- a/app/models/lfs_object.rb
+++ b/app/models/lfs_object.rb
@@ -15,7 +15,7 @@ class LfsObject < ActiveRecord::Base
after_save if: :file_changed?, on: [:create, :update] do
run_after_commit do
- file.schedule_migration_to_object_storage
+ file.schedule_background_upload
end
end
diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb
index d80221144b4..06842a58571 100644
--- a/app/uploaders/job_artifact_uploader.rb
+++ b/app/uploaders/job_artifact_uploader.rb
@@ -1,5 +1,4 @@
class JobArtifactUploader < GitlabUploader
- prepend EE::JobArtifactUploader
extend Workhorse::UploadPath
include ObjectStorage::Concern
diff --git a/db/schema.rb b/db/schema.rb
index 46f2558e7ca..5ce6a015e83 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20180213131630) do
+ActiveRecord::Schema.define(version: 20180216121030) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
diff --git a/spec/ee/spec/finders/geo/attachment_registry_finder_spec.rb b/spec/ee/spec/finders/geo/attachment_registry_finder_spec.rb
deleted file mode 100644
index 9f0f5f2ab87..00000000000
--- a/spec/ee/spec/finders/geo/attachment_registry_finder_spec.rb
+++ /dev/null
@@ -1,270 +0,0 @@
-require 'spec_helper'
-
-describe Geo::AttachmentRegistryFinder, :geo do
- include ::EE::GeoHelpers
-
- let(:secondary) { create(:geo_node) }
-
- let(:synced_group) { create(:group) }
- let(:synced_subgroup) { create(:group, parent: synced_group) }
- let(:unsynced_group) { create(:group) }
- let(:synced_project) { create(:project, group: synced_group) }
- let(:unsynced_project) { create(:project, group: unsynced_group, repository_storage: 'broken') }
-
- let!(:upload_1) { create(:upload, model: synced_group) }
- let!(:upload_2) { create(:upload, model: unsynced_group) }
- let!(:upload_3) { create(:upload, :issuable_upload, model: synced_project) }
- let!(:upload_4) { create(:upload, model: unsynced_project) }
- let(:upload_5) { create(:upload, model: synced_project) }
- let(:upload_6) { create(:upload, :personal_snippet_upload) }
- let(:upload_7) { create(:upload, model: synced_subgroup) }
- let(:lfs_object) { create(:lfs_object) }
-
- subject { described_class.new(current_node: secondary) }
-
- before do
- stub_current_geo_node(secondary)
- end
-
- # Disable transactions via :delete method because a foreign table
- # can't see changes inside a transaction of a different connection.
- context 'FDW', :delete do
- before do
- skip('FDW is not configured') if Gitlab::Database.postgresql? && !Gitlab::Geo.fdw?
- end
-
- describe '#find_synced_attachments' do
- it 'delegates to #fdw_find_synced_attachments' do
- expect(subject).to receive(:fdw_find_synced_attachments).and_call_original
-
- subject.find_synced_attachments
- end
-
- it 'returns synced avatars, attachment, personal snippets and files' do
- create(:geo_file_registry, :avatar, file_id: upload_1.id)
- create(:geo_file_registry, :avatar, file_id: upload_2.id)
- create(:geo_file_registry, :avatar, file_id: upload_3.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_6.id)
- create(:geo_file_registry, :avatar, file_id: upload_7.id)
- create(:geo_file_registry, :lfs, file_id: lfs_object.id)
-
- synced_attachments = subject.find_synced_attachments
-
- expect(synced_attachments.pluck(:id)).to match_array([upload_1.id, upload_2.id, upload_6.id, upload_7.id])
- end
-
- context 'with selective sync' do
- it 'falls back to legacy queries' do
- secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
-
- expect(subject).to receive(:legacy_find_synced_attachments)
-
- subject.find_synced_attachments
- end
- end
- end
-
- describe '#find_failed_attachments' do
- it 'delegates to #fdw_find_failed_attachments' do
- expect(subject).to receive(:fdw_find_failed_attachments).and_call_original
-
- subject.find_failed_attachments
- end
-
- it 'returns failed avatars, attachment, personal snippets and files' do
- create(:geo_file_registry, :avatar, file_id: upload_1.id)
- create(:geo_file_registry, :avatar, file_id: upload_2.id)
- create(:geo_file_registry, :avatar, file_id: upload_3.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_6.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_7.id, success: false)
- create(:geo_file_registry, :lfs, file_id: lfs_object.id, success: false)
-
- failed_attachments = subject.find_failed_attachments
-
- expect(failed_attachments.pluck(:id)).to match_array([upload_3.id, upload_6.id, upload_7.id])
- end
-
- context 'with selective sync' do
- it 'falls back to legacy queries' do
- secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
-
- expect(subject).to receive(:legacy_find_failed_attachments)
-
- subject.find_failed_attachments
- end
- end
- end
-
- describe '#find_unsynced_attachments' do
- it 'delegates to #fdw_find_unsynced_attachments' do
- expect(subject).to receive(:fdw_find_unsynced_attachments).and_call_original
-
- subject.find_unsynced_attachments(batch_size: 10)
- end
-
- it 'returns uploads without an entry on the tracking database' do
- create(:geo_file_registry, :avatar, file_id: upload_1.id, success: true)
-
- uploads = subject.find_unsynced_attachments(batch_size: 10)
-
- expect(uploads.map(&:id)).to match_array([upload_2.id, upload_3.id, upload_4.id])
- end
-
- it 'excludes uploads without an entry on the tracking database' do
- create(:geo_file_registry, :avatar, file_id: upload_1.id, success: true)
-
- uploads = subject.find_unsynced_attachments(batch_size: 10, except_registry_ids: [upload_2.id])
-
- expect(uploads.map(&:id)).to match_array([upload_3.id, upload_4.id])
- end
- end
- end
-
- context 'Legacy' do
- before do
- allow(Gitlab::Geo).to receive(:fdw?).and_return(false)
- end
-
- describe '#find_synced_attachments' do
- it 'delegates to #legacy_find_synced_attachments' do
- expect(subject).to receive(:legacy_find_synced_attachments).and_call_original
-
- subject.find_synced_attachments
- end
-
- it 'returns synced avatars, attachment, personal snippets and files' do
- create(:geo_file_registry, :avatar, file_id: upload_1.id)
- create(:geo_file_registry, :avatar, file_id: upload_2.id)
- create(:geo_file_registry, :avatar, file_id: upload_3.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_6.id)
- create(:geo_file_registry, :avatar, file_id: upload_7.id)
- create(:geo_file_registry, :lfs, file_id: lfs_object.id)
-
- synced_attachments = subject.find_synced_attachments
-
- expect(synced_attachments).to match_array([upload_1, upload_2, upload_6, upload_7])
- end
-
- context 'with selective sync by namespace' do
- it 'returns synced avatars, attachment, personal snippets and files' do
- create(:geo_file_registry, :avatar, file_id: upload_1.id)
- create(:geo_file_registry, :avatar, file_id: upload_2.id)
- create(:geo_file_registry, :avatar, file_id: upload_3.id)
- create(:geo_file_registry, :avatar, file_id: upload_4.id)
- create(:geo_file_registry, :avatar, file_id: upload_5.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_6.id)
- create(:geo_file_registry, :avatar, file_id: upload_7.id)
- create(:geo_file_registry, :lfs, file_id: lfs_object.id)
-
- secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
-
- synced_attachments = subject.find_synced_attachments
-
- expect(synced_attachments).to match_array([upload_1, upload_3, upload_6, upload_7])
- end
- end
-
- context 'with selective sync by shard' do
- it 'returns synced avatars, attachment, personal snippets and files' do
- create(:geo_file_registry, :avatar, file_id: upload_1.id)
- create(:geo_file_registry, :avatar, file_id: upload_2.id)
- create(:geo_file_registry, :avatar, file_id: upload_3.id)
- create(:geo_file_registry, :avatar, file_id: upload_4.id)
- create(:geo_file_registry, :avatar, file_id: upload_5.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_6.id)
- create(:geo_file_registry, :avatar, file_id: upload_7.id)
- create(:geo_file_registry, :lfs, file_id: lfs_object.id)
-
- secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['default'])
-
- synced_attachments = subject.find_synced_attachments
-
- expect(synced_attachments).to match_array([upload_1, upload_3, upload_6])
- end
- end
- end
-
- describe '#find_failed_attachments' do
- it 'delegates to #legacy_find_failed_attachments' do
- expect(subject).to receive(:legacy_find_failed_attachments).and_call_original
-
- subject.find_failed_attachments
- end
-
- it 'returns failed avatars, attachment, personal snippets and files' do
- create(:geo_file_registry, :avatar, file_id: upload_1.id)
- create(:geo_file_registry, :avatar, file_id: upload_2.id)
- create(:geo_file_registry, :avatar, file_id: upload_3.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_6.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_7.id, success: false)
- create(:geo_file_registry, :lfs, file_id: lfs_object.id, success: false)
-
- failed_attachments = subject.find_failed_attachments
-
- expect(failed_attachments).to match_array([upload_3, upload_6, upload_7])
- end
-
- context 'with selective sync by namespace' do
- it 'returns failed avatars, attachment, personal snippets and files' do
- create(:geo_file_registry, :avatar, file_id: upload_1.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_2.id)
- create(:geo_file_registry, :avatar, file_id: upload_3.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_4.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_5.id)
- create(:geo_file_registry, :avatar, file_id: upload_6.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_7.id, success: false)
- create(:geo_file_registry, :lfs, file_id: lfs_object.id, success: false)
-
- secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
-
- failed_attachments = subject.find_failed_attachments
-
- expect(failed_attachments).to match_array([upload_1, upload_3, upload_6, upload_7])
- end
- end
-
- context 'with selective sync by shard' do
- it 'returns failed avatars, attachment, personal snippets and files' do
- create(:geo_file_registry, :avatar, file_id: upload_1.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_2.id)
- create(:geo_file_registry, :avatar, file_id: upload_3.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_4.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_5.id)
- create(:geo_file_registry, :avatar, file_id: upload_6.id, success: false)
- create(:geo_file_registry, :avatar, file_id: upload_7.id, success: false)
- create(:geo_file_registry, :lfs, file_id: lfs_object.id, success: false)
-
- secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['default'])
-
- failed_attachments = subject.find_failed_attachments
-
- expect(failed_attachments).to match_array([upload_1, upload_3, upload_6])
- end
- end
- end
-
- describe '#find_unsynced_attachments' do
- it 'delegates to #legacy_find_unsynced_attachments' do
- expect(subject).to receive(:legacy_find_unsynced_attachments).and_call_original
-
- subject.find_unsynced_attachments(batch_size: 10)
- end
-
- it 'returns LFS objects without an entry on the tracking database' do
- create(:geo_file_registry, :avatar, file_id: upload_1.id, success: true)
-
- uploads = subject.find_unsynced_attachments(batch_size: 10)
-
- expect(uploads).to match_array([upload_2, upload_3, upload_4])
- end
-
- it 'excludes uploads without an entry on the tracking database' do
- create(:geo_file_registry, :avatar, file_id: upload_1.id, success: true)
-
- uploads = subject.find_unsynced_attachments(batch_size: 10, except_registry_ids: [upload_2.id])
-
- expect(uploads).to match_array([upload_3, upload_4])
- end
- end
- end
-end
diff --git a/spec/ee/spec/lib/gitlab/geo/file_transfer_spec.rb b/spec/ee/spec/lib/gitlab/geo/file_transfer_spec.rb
deleted file mode 100644
index 4cb2a1ec08f..00000000000
--- a/spec/ee/spec/lib/gitlab/geo/file_transfer_spec.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-require 'spec_helper'
-
-describe Gitlab::Geo::FileTransfer do
- let(:user) { create(:user, avatar: fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/png')) }
- let(:upload) { Upload.find_by(model: user, uploader: 'AvatarUploader') }
-
- subject { described_class.new(:file, upload) }
-
- describe '#execute' do
- context 'user avatar' do
- it 'sets an absolute path' do
- expect(subject.file_type).to eq(:file)
- expect(subject.file_id).to eq(upload.id)
- expect(subject.filename).to eq(upload.absolute_path)
- expect(Pathname.new(subject.filename).absolute?).to be_truthy
- expect(subject.request_data).to eq({ id: upload.model_id,
- type: 'User',
- checksum: upload.checksum })
- end
- end
- end
-end
diff --git a/spec/ee/spec/lib/gitlab/geo/log_cursor/daemon_spec.rb b/spec/ee/spec/lib/gitlab/geo/log_cursor/daemon_spec.rb
deleted file mode 100644
index af475a966a0..00000000000
--- a/spec/ee/spec/lib/gitlab/geo/log_cursor/daemon_spec.rb
+++ /dev/null
@@ -1,414 +0,0 @@
-require 'spec_helper'
-
-describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared_state do
- include ::EE::GeoHelpers
-
- set(:primary) { create(:geo_node, :primary) }
- set(:secondary) { create(:geo_node) }
-
- let(:options) { {} }
- subject(:daemon) { described_class.new(options) }
-
- around do |example|
- Sidekiq::Testing.fake! { example.run }
- end
-
- before do
- stub_current_geo_node(secondary)
-
- allow(daemon).to receive(:trap_signals)
- allow(daemon).to receive(:arbitrary_sleep).and_return(0.1)
- end
-
- describe '#run!' do
- it 'traps signals' do
- is_expected.to receive(:exit?).and_return(true)
- is_expected.to receive(:trap_signals)
-
- daemon.run!
- end
-
- it 'delegates to #run_once! in a loop' do
- is_expected.to receive(:exit?).and_return(false, false, false, true)
- is_expected.to receive(:run_once!).twice
-
- daemon.run!
- end
-
- it 'skips execution if cannot achieve a lease' do
- is_expected.to receive(:exit?).and_return(false, true)
- is_expected.not_to receive(:run_once!)
- expect_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain_with_ttl).and_return({ ttl: 1, uuid: false })
-
- daemon.run!
- end
-
- it 'skips execution if not a Geo node' do
- stub_current_geo_node(nil)
-
- is_expected.to receive(:exit?).and_return(false, true)
- is_expected.to receive(:sleep).with(1.minute)
- is_expected.not_to receive(:run_once!)
-
- daemon.run!
- end
-
- it 'skips execution if the current node is a primary' do
- stub_current_geo_node(primary)
-
- is_expected.to receive(:exit?).and_return(false, true)
- is_expected.to receive(:sleep).with(1.minute)
- is_expected.not_to receive(:run_once!)
-
- daemon.run!
- end
- end
-
- describe '#run_once!' do
- context 'when replaying a repository created event' do
- let(:project) { create(:project) }
- let(:repository_created_event) { create(:geo_repository_created_event, project: project) }
- let(:event_log) { create(:geo_event_log, repository_created_event: repository_created_event) }
- let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
-
- it 'creates a new project registry' do
- expect { daemon.run_once! }.to change(Geo::ProjectRegistry, :count).by(1)
- end
-
- it 'sets resync attributes to true' do
- daemon.run_once!
-
- registry = Geo::ProjectRegistry.last
-
- expect(registry).to have_attributes(project_id: project.id, resync_repository: true, resync_wiki: true)
- end
-
- it 'sets resync_wiki to false if wiki_path is nil' do
- repository_created_event.update!(wiki_path: nil)
-
- daemon.run_once!
-
- registry = Geo::ProjectRegistry.last
-
- expect(registry).to have_attributes(project_id: project.id, resync_repository: true, resync_wiki: false)
- end
-
- it 'performs Geo::ProjectSyncWorker' do
- expect(Geo::ProjectSyncWorker).to receive(:perform_async)
- .with(project.id, anything).once
-
- daemon.run_once!
- end
- end
-
- context 'when replaying a repository updated event' do
- let(:project) { create(:project) }
- let(:repository_updated_event) { create(:geo_repository_updated_event, project: project) }
- let(:event_log) { create(:geo_event_log, repository_updated_event: repository_updated_event) }
- let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
-
- it 'creates a new project registry if it does not exist' do
- expect { daemon.run_once! }.to change(Geo::ProjectRegistry, :count).by(1)
- end
-
- it 'sets resync_repository to true if event source is repository' do
- repository_updated_event.update!(source: Geo::RepositoryUpdatedEvent::REPOSITORY)
- registry = create(:geo_project_registry, :synced, project: repository_updated_event.project)
-
- daemon.run_once!
-
- expect(registry.reload.resync_repository).to be true
- end
-
- it 'sets resync_wiki to true if event source is wiki' do
- repository_updated_event.update!(source: Geo::RepositoryUpdatedEvent::WIKI)
- registry = create(:geo_project_registry, :synced, project: repository_updated_event.project)
-
- daemon.run_once!
-
- expect(registry.reload.resync_wiki).to be true
- end
-
- it 'performs Geo::ProjectSyncWorker' do
- expect(Geo::ProjectSyncWorker).to receive(:perform_async)
- .with(project.id, anything).once
-
- daemon.run_once!
- end
- end
-
- context 'when replaying a repository deleted event' do
- let(:event_log) { create(:geo_event_log, :deleted_event) }
- let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
- let(:repository_deleted_event) { event_log.repository_deleted_event }
- let(:project) { repository_deleted_event.project }
-
- it 'does not create a tracking database entry' do
- expect { daemon.run_once! }.not_to change(Geo::ProjectRegistry, :count)
- end
-
- it 'schedules a GeoRepositoryDestroyWorker' do
- project_id = repository_deleted_event.project_id
- project_name = repository_deleted_event.deleted_project_name
- project_path = repository_deleted_event.deleted_path
-
- expect(::GeoRepositoryDestroyWorker).to receive(:perform_async)
- .with(project_id, project_name, project_path, project.repository_storage)
-
- daemon.run_once!
- end
-
- it 'removes the tracking database entry if exist' do
- create(:geo_project_registry, :synced, project: project)
-
- expect { daemon.run_once! }.to change(Geo::ProjectRegistry, :count).by(-1)
- end
- end
-
- context 'when replaying a repositories changed event' do
- let(:repositories_changed_event) { create(:geo_repositories_changed_event, geo_node: secondary) }
- let(:event_log) { create(:geo_event_log, repositories_changed_event: repositories_changed_event) }
- let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
-
- it 'schedules a GeoRepositoryDestroyWorker when event node is the current node' do
- expect(Geo::RepositoriesCleanUpWorker).to receive(:perform_in).with(within(5.minutes).of(1.hour), secondary.id)
-
- daemon.run_once!
- end
-
- it 'does not schedule a GeoRepositoryDestroyWorker when event node is not the current node' do
- stub_current_geo_node(build(:geo_node))
-
- expect(Geo::RepositoriesCleanUpWorker).not_to receive(:perform_in)
-
- daemon.run_once!
- end
- end
-
- context 'when node has namespace restrictions' do
- let(:group_1) { create(:group) }
- let(:group_2) { create(:group) }
- let(:project) { create(:project, group: group_1) }
- let(:repository_updated_event) { create(:geo_repository_updated_event, project: project) }
- let(:event_log) { create(:geo_event_log, repository_updated_event: repository_updated_event) }
- let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
-
- before do
- allow(Geo::ProjectSyncWorker).to receive(:perform_async)
- end
-
- it 'replays events for projects that belong to selected namespaces to replicate' do
- secondary.update!(namespaces: [group_1])
-
- expect { daemon.run_once! }.to change(Geo::ProjectRegistry, :count).by(1)
- end
-
- it 'does not replay events for projects that do not belong to selected namespaces to replicate' do
- secondary.update!(selective_sync_type: 'namespaces', namespaces: [group_2])
-
- expect { daemon.run_once! }.not_to change(Geo::ProjectRegistry, :count)
- end
-
- it 'does not replay events for projects that do not belong to selected shards to replicate' do
- secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
-
- expect { daemon.run_once! }.not_to change(Geo::ProjectRegistry, :count)
- end
- end
-
- context 'when processing a repository renamed event' do
- let(:event_log) { create(:geo_event_log, :renamed_event) }
- let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
- let(:repository_renamed_event) { event_log.repository_renamed_event }
-
- it 'does not create a new project registry' do
- expect { daemon.run_once! }.not_to change(Geo::ProjectRegistry, :count)
- end
-
- it 'schedules a Geo::RenameRepositoryWorker' do
- project_id = repository_renamed_event.project_id
- old_path_with_namespace = repository_renamed_event.old_path_with_namespace
- new_path_with_namespace = repository_renamed_event.new_path_with_namespace
-
- expect(::Geo::RenameRepositoryWorker).to receive(:perform_async)
- .with(project_id, old_path_with_namespace, new_path_with_namespace)
-
- daemon.run_once!
- end
- end
-
- context 'when processing a hashed storage migration event' do
- let(:event_log) { create(:geo_event_log, :hashed_storage_migration_event) }
- let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
- let(:hashed_storage_migrated_event) { event_log.hashed_storage_migrated_event }
-
- it 'does not create a new project registry' do
- expect { daemon.run_once! }.not_to change(Geo::ProjectRegistry, :count)
- end
-
- it 'schedules a Geo::HashedStorageMigrationWorker' do
- project = hashed_storage_migrated_event.project
- old_disk_path = hashed_storage_migrated_event.old_disk_path
- new_disk_path = hashed_storage_migrated_event.new_disk_path
- old_storage_version = project.storage_version
-
- expect(::Geo::HashedStorageMigrationWorker).to receive(:perform_async)
- .with(project.id, old_disk_path, new_disk_path, old_storage_version)
-
- daemon.run_once!
- end
- end
-
- context 'when processing an attachment migration event to hashed storage' do
- let(:event_log) { create(:geo_event_log, :hashed_storage_attachments_event) }
- let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
- let(:hashed_storage_attachments_event) { event_log.hashed_storage_attachments_event }
-
- it 'does not create a new project registry' do
- expect { daemon.run_once! }.not_to change(Geo::ProjectRegistry, :count)
- end
-
- it 'schedules a Geo::HashedStorageAttachmentsMigrationWorker' do
- project = hashed_storage_attachments_event.project
- old_attachments_path = hashed_storage_attachments_event.old_attachments_path
- new_attachments_path = hashed_storage_attachments_event.new_attachments_path
-
- expect(::Geo::HashedStorageAttachmentsMigrationWorker).to receive(:perform_async)
- .with(project.id, old_attachments_path, new_attachments_path)
-
- daemon.run_once!
- end
- end
-
- context 'when replaying a LFS object deleted event' do
- let(:event_log) { create(:geo_event_log, :lfs_object_deleted_event) }
- let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
- let(:lfs_object_deleted_event) { event_log.lfs_object_deleted_event }
- let(:lfs_object) { lfs_object_deleted_event.lfs_object }
-
- it 'does not create a tracking database entry' do
- expect { daemon.run_once! }.not_to change(Geo::FileRegistry, :count)
- end
-
- it 'schedules a Geo::FileRemovalWorker' do
- file_path = File.join(LfsObjectUploader.root, lfs_object_deleted_event.file_path)
-
- expect(::Geo::FileRemovalWorker).to receive(:perform_async)
- .with(file_path)
-
- daemon.run_once!
- end
-
- it 'removes the tracking database entry if exist' do
- create(:geo_file_registry, :lfs, file_id: lfs_object.id)
-
- expect { daemon.run_once! }.to change(Geo::FileRegistry.lfs_objects, :count).by(-1)
- end
- end
-
- context 'when replaying a job artifact event' do
- let(:event_log) { create(:geo_event_log, :job_artifact_deleted_event) }
- let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
- let(:job_artifact_deleted_event) { event_log.job_artifact_deleted_event }
- let(:job_artifact) { job_artifact_deleted_event.job_artifact }
-
- context 'with a tracking database entry' do
- before do
- create(:geo_file_registry, :job_artifact, file_id: job_artifact.id)
- end
-
- context 'with a file' do
- context 'when the delete succeeds' do
- it 'removes the tracking database entry' do
- expect { daemon.run_once! }.to change(Geo::FileRegistry.job_artifacts, :count).by(-1)
- end
-
- it 'deletes the file' do
- expect { daemon.run_once! }.to change { File.exist?(job_artifact.file.path) }.from(true).to(false)
- end
- end
-
- context 'when the delete fails' do
- before do
- expect(daemon).to receive(:delete_file).and_return(false)
- end
-
- it 'does not remove the tracking database entry' do
- expect { daemon.run_once! }.not_to change(Geo::FileRegistry.job_artifacts, :count)
- end
- end
- end
-
- context 'without a file' do
- before do
- FileUtils.rm(job_artifact.file.path)
- end
-
- it 'removes the tracking database entry' do
- expect { daemon.run_once! }.to change(Geo::FileRegistry.job_artifacts, :count).by(-1)
- end
- end
- end
-
- context 'without a tracking database entry' do
- it 'does not create a tracking database entry' do
- expect { daemon.run_once! }.not_to change(Geo::FileRegistry, :count)
- end
-
- it 'does not delete the file (yet, due to possible race condition)' do
- expect { daemon.run_once! }.not_to change { File.exist?(job_artifact.file.path) }.from(true)
- end
- end
- end
- end
-
- describe '#delete_file' do
- context 'when the file exists' do
- let!(:file) { fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png") }
-
- context 'when the delete does not raise an exception' do
- it 'returns true' do
- expect(daemon.send(:delete_file, file.path)).to be_truthy
- end
-
- it 'does not log an error' do
- expect(daemon).not_to receive(:logger)
-
- daemon.send(:delete_file, file.path)
- end
- end
-
- context 'when the delete raises an exception' do
- before do
- expect(File).to receive(:delete).and_raise('something went wrong')
- end
-
- it 'returns false' do
- expect(daemon.send(:delete_file, file.path)).to be_falsey
- end
-
- it 'logs an error' do
- logger = double(logger)
- expect(daemon).to receive(:logger).and_return(logger)
- expect(logger).to receive(:error).with('Failed to remove file', exception: 'RuntimeError', details: 'something went wrong', filename: file.path)
-
- daemon.send(:delete_file, file.path)
- end
- end
- end
-
- context 'when the file does not exist' do
- it 'returns false' do
- expect(daemon.send(:delete_file, '/does/not/exist')).to be_falsey
- end
-
- it 'logs an error' do
- logger = double(logger)
- expect(daemon).to receive(:logger).and_return(logger)
- expect(logger).to receive(:error).with('Failed to remove file', exception: 'Errno::ENOENT', details: 'No such file or directory @ unlink_internal - /does/not/exist', filename: '/does/not/exist')
-
- daemon.send(:delete_file, '/does/not/exist')
- end
- end
- end
-end
diff --git a/spec/ee/spec/services/geo/file_download_service_spec.rb b/spec/ee/spec/services/geo/file_download_service_spec.rb
deleted file mode 100644
index 4fb0d89dbde..00000000000
--- a/spec/ee/spec/services/geo/file_download_service_spec.rb
+++ /dev/null
@@ -1,227 +0,0 @@
-require 'spec_helper'
-
-describe Geo::FileDownloadService do
- include ::EE::GeoHelpers
-
- set(:primary) { create(:geo_node, :primary) }
- set(:secondary) { create(:geo_node) }
-
- before do
- stub_current_geo_node(secondary)
-
- allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true)
- end
-
- describe '#execute' do
- context 'user avatar' do
- let(:user) { create(:user, avatar: fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/png')) }
- let(:upload) { Upload.find_by(model: user, uploader: 'AvatarUploader') }
-
- subject(:execute!) { described_class.new(:avatar, upload.id).execute }
-
- it 'downloads a user avatar' do
- stub_transfer(Gitlab::Geo::FileTransfer, 100)
-
- expect { execute! }.to change { Geo::FileRegistry.synced.count }.by(1)
- end
-
- it 'registers when the download fails' do
- stub_transfer(Gitlab::Geo::FileTransfer, -1)
-
- expect { execute! }.to change { Geo::FileRegistry.failed.count }.by(1)
- expect(Geo::FileRegistry.last.retry_count).to eq(1)
- expect(Geo::FileRegistry.last.retry_at).to be_present
- end
-
- it 'registers when the download fails with some other error' do
- stub_transfer(Gitlab::Geo::FileTransfer, nil)
-
- expect { execute! }.to change { Geo::FileRegistry.failed.count }.by(1)
- end
- end
-
- context 'group avatar' do
- let(:group) { create(:group, avatar: fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/png')) }
- let(:upload) { Upload.find_by(model: group, uploader: 'AvatarUploader') }
-
- subject(:execute!) { described_class.new(:avatar, upload.id).execute }
-
- it 'downloads a group avatar' do
- stub_transfer(Gitlab::Geo::FileTransfer, 100)
-
- expect { execute! }.to change { Geo::FileRegistry.synced.count }.by(1)
- end
-
- it 'registers when the download fails' do
- stub_transfer(Gitlab::Geo::FileTransfer, -1)
-
- expect { execute! }.to change { Geo::FileRegistry.failed.count }.by(1)
- end
- end
-
- context 'project avatar' do
- let(:project) { create(:project, avatar: fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/png')) }
- let(:upload) { Upload.find_by(model: project, uploader: 'AvatarUploader') }
-
- subject(:execute!) { described_class.new(:avatar, upload.id).execute }
-
- it 'downloads a project avatar' do
- stub_transfer(Gitlab::Geo::FileTransfer, 100)
-
- expect { execute! }.to change { Geo::FileRegistry.synced.count }.by(1)
- end
-
- it 'registers when the download fails' do
- stub_transfer(Gitlab::Geo::FileTransfer, -1)
-
- expect { execute! }.to change { Geo::FileRegistry.failed.count }.by(1)
- end
- end
-
- context 'with an attachment' do
- let(:note) { create(:note, :with_attachment) }
- let(:upload) { Upload.find_by(model: note, uploader: 'AttachmentUploader') }
-
- subject(:execute!) { described_class.new(:attachment, upload.id).execute }
-
- it 'downloads the attachment' do
- stub_transfer(Gitlab::Geo::FileTransfer, 100)
-
- expect { execute! }.to change { Geo::FileRegistry.synced.count }.by(1)
- end
-
- it 'registers when the download fails' do
- stub_transfer(Gitlab::Geo::FileTransfer, -1)
-
- expect { execute! }.to change { Geo::FileRegistry.failed.count }.by(1)
- end
- end
-
- context 'with a snippet' do
- let(:upload) { create(:upload, :personal_snippet_upload) }
-
- subject(:execute!) { described_class.new(:personal_file, upload.id).execute }
-
- it 'downloads the file' do
- stub_transfer(Gitlab::Geo::FileTransfer, 100)
-
- expect { execute! }.to change { Geo::FileRegistry.synced.count }.by(1)
- end
-
- it 'registers when the download fails' do
- stub_transfer(Gitlab::Geo::FileTransfer, -1)
-
- expect { execute! }.to change { Geo::FileRegistry.failed.count }.by(1)
- end
- end
-
- context 'with file upload' do
- let(:project) { create(:project) }
- let(:upload) { Upload.find_by(model: project, uploader: 'FileUploader') }
-
- subject { described_class.new(:file, upload.id) }
-
- before do
- FileUploader.new(project).store!(fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/png'))
- end
-
- it 'downloads the file' do
- stub_transfer(Gitlab::Geo::FileTransfer, 100)
-
- expect { subject.execute }.to change { Geo::FileRegistry.synced.count }.by(1)
- end
-
- it 'registers when the download fails' do
- stub_transfer(Gitlab::Geo::FileTransfer, -1)
-
- expect { subject.execute }.to change { Geo::FileRegistry.failed.count }.by(1)
- end
- end
-
- context 'with namespace file upload' do
- let(:group) { create(:group) }
- let(:upload) { Upload.find_by(model: group, uploader: 'NamespaceFileUploader') }
-
- subject { described_class.new(:file, upload.id) }
-
- before do
- NamespaceFileUploader.new(group).store!(fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/png'))
- end
-
- it 'downloads the file' do
- stub_transfer(Gitlab::Geo::FileTransfer, 100)
-
- expect { subject.execute }.to change { Geo::FileRegistry.synced.count }.by(1)
- end
-
- it 'registers when the download fails' do
- stub_transfer(Gitlab::Geo::FileTransfer, -1)
-
- expect { subject.execute }.to change { Geo::FileRegistry.failed.count }.by(1)
- end
- end
-
- context 'LFS object' do
- let(:lfs_object) { create(:lfs_object) }
-
- subject { described_class.new(:lfs, lfs_object.id) }
-
- it 'downloads an LFS object' do
- stub_transfer(Gitlab::Geo::LfsTransfer, 100)
-
- expect { subject.execute }.to change { Geo::FileRegistry.synced.count }.by(1)
- end
-
- it 'registers when the download fails' do
- stub_transfer(Gitlab::Geo::LfsTransfer, -1)
-
- expect { subject.execute }.to change { Geo::FileRegistry.failed.count }.by(1)
- end
-
- it 'logs a message' do
- stub_transfer(Gitlab::Geo::LfsTransfer, 100)
-
- expect(Gitlab::Geo::Logger).to receive(:info).with(hash_including(:message, :download_time_s, success: true, bytes_downloaded: 100)).and_call_original
-
- subject.execute
- end
- end
-
- context 'job artifacts' do
- let(:job_artifact) { create(:ci_job_artifact) }
-
- subject { described_class.new(:job_artifact, job_artifact.id) }
-
- it 'downloads a job artifact' do
- stub_transfer(Gitlab::Geo::JobArtifactTransfer, 100)
-
- expect { subject.execute }.to change { Geo::FileRegistry.synced.count }.by(1)
- end
-
- it 'registers when the download fails' do
- stub_transfer(Gitlab::Geo::JobArtifactTransfer, -1)
-
- expect { subject.execute }.to change { Geo::FileRegistry.failed.count }.by(1)
- end
-
- it 'logs a message' do
- stub_transfer(Gitlab::Geo::JobArtifactTransfer, 100)
-
- expect(Gitlab::Geo::Logger).to receive(:info).with(hash_including(:message, :download_time_s, success: true, bytes_downloaded: 100)).and_call_original
-
- subject.execute
- end
- end
-
- context 'bad object type' do
- it 'raises an error' do
- expect { described_class.new(:bad, 1).execute }.to raise_error(NameError)
- end
- end
-
- def stub_transfer(kls, result)
- instance = double("(instance of #{kls})", download_from_primary: result)
- allow(kls).to receive(:new).and_return(instance)
- end
- end
-end
diff --git a/spec/ee/spec/services/geo/files_expire_service_spec.rb b/spec/ee/spec/services/geo/files_expire_service_spec.rb
deleted file mode 100644
index 09b0b386ed1..00000000000
--- a/spec/ee/spec/services/geo/files_expire_service_spec.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-require 'spec_helper'
-
-# Disable transactions via :delete method because a foreign table
-# can't see changes inside a transaction of a different connection.
-describe Geo::FilesExpireService, :geo, :delete do
- let(:project) { create(:project) }
- let!(:old_full_path) { project.full_path }
- subject { described_class.new(project, old_full_path) }
-
- describe '#execute' do
- let(:file_uploader) { build(:file_uploader, project: project) }
- let!(:upload) { Upload.find_by(path: file_uploader.upload_path) }
- let!(:file_registry) { create(:geo_file_registry, file_id: upload.id) }
-
- before do
- project.update(path: "#{project.path}_renamed")
- end
-
- context 'when in Geo secondary node' do
- before do
- allow(Gitlab::Geo).to receive(:secondary?) { true }
- end
-
- it 'remove file from disk' do
- file_path = File.join(subject.base_dir, upload.path)
- expect(File.exist?(file_path)).to be_truthy
-
- Sidekiq::Testing.inline! { subject.execute }
-
- expect(File.exist?(file_path)).to be_falsey
- end
-
- it 'removes file_registry associates with upload' do
- expect(file_registry.success).to be_truthy
-
- subject.execute
-
- expect { file_registry.reload }.to raise_error(ActiveRecord::RecordNotFound)
- end
- end
-
- context 'when not in Geo secondary node' do
- it 'no-op execute action' do
- expect(subject).not_to receive(:schedule_file_removal)
- expect(subject).not_to receive(:mark_for_resync!)
-
- subject.execute
- end
- end
- end
-end
diff --git a/spec/ee/spec/services/geo/hashed_storage_attachments_migration_service_spec.rb b/spec/ee/spec/services/geo/hashed_storage_attachments_migration_service_spec.rb
deleted file mode 100644
index 40e06705cf5..00000000000
--- a/spec/ee/spec/services/geo/hashed_storage_attachments_migration_service_spec.rb
+++ /dev/null
@@ -1,83 +0,0 @@
-require 'spec_helper'
-
-def base_path(storage)
- File.join(FileUploader.root, storage.disk_path)
-end
-
-describe Geo::HashedStorageAttachmentsMigrationService do
- let!(:project) { create(:project) }
-
- let(:legacy_storage) { Storage::LegacyProject.new(project) }
- let(:hashed_storage) { Storage::HashedProject.new(project) }
-
- let!(:upload) { Upload.find_by(path: file_uploader.upload_path) }
- let(:file_uploader) { build(:file_uploader, project: project) }
- let(:old_path) { File.join(base_path(legacy_storage), upload.path) }
- let(:new_path) { File.join(base_path(hashed_storage), upload.path) }
-
- subject(:service) do
- described_class.new(project.id,
- old_attachments_path: legacy_storage.disk_path,
- new_attachments_path: hashed_storage.disk_path)
- end
-
- describe '#execute' do
- context 'when succeeds' do
- it 'moves attachments to hashed storage layout' do
- expect(File.file?(old_path)).to be_truthy
- expect(File.file?(new_path)).to be_falsey
- expect(File.exist?(base_path(legacy_storage))).to be_truthy
- expect(File.exist?(base_path(hashed_storage))).to be_falsey
- expect(FileUtils).to receive(:mv).with(base_path(legacy_storage), base_path(hashed_storage)).and_call_original
-
- service.execute
-
- expect(File.exist?(base_path(hashed_storage))).to be_truthy
- expect(File.exist?(base_path(legacy_storage))).to be_falsey
- expect(File.file?(old_path)).to be_falsey
- expect(File.file?(new_path)).to be_truthy
- end
- end
-
- context 'when original folder does not exist anymore' do
- before do
- FileUtils.rm_rf(base_path(legacy_storage))
- end
-
- it 'skips moving folders and go to next' do
- expect(FileUtils).not_to receive(:mv).with(base_path(legacy_storage), base_path(hashed_storage))
-
- service.execute
-
- expect(File.exist?(base_path(hashed_storage))).to be_falsey
- expect(File.file?(new_path)).to be_falsey
- end
- end
-
- context 'when target folder already exists' do
- before do
- FileUtils.mkdir_p(base_path(hashed_storage))
- end
-
- it 'raises AttachmentMigrationError' do
- expect(FileUtils).not_to receive(:mv).with(base_path(legacy_storage), base_path(hashed_storage))
-
- expect { service.execute }.to raise_error(::Geo::AttachmentMigrationError)
- end
- end
- end
-
- describe '#async_execute' do
- it 'starts the worker' do
- expect(Geo::HashedStorageAttachmentsMigrationWorker).to receive(:perform_async)
-
- service.async_execute
- end
-
- it 'returns job id' do
- allow(Geo::HashedStorageAttachmentsMigrationWorker).to receive(:perform_async).and_return('foo')
-
- expect(service.async_execute).to eq('foo')
- end
- end
-end
diff --git a/spec/ee/spec/workers/geo/file_download_dispatch_worker_spec.rb b/spec/ee/spec/workers/geo/file_download_dispatch_worker_spec.rb
deleted file mode 100644
index ad7cad3128a..00000000000
--- a/spec/ee/spec/workers/geo/file_download_dispatch_worker_spec.rb
+++ /dev/null
@@ -1,291 +0,0 @@
-require 'spec_helper'
-
-describe Geo::FileDownloadDispatchWorker, :geo do
- include ::EE::GeoHelpers
-
- let(:primary) { create(:geo_node, :primary, host: 'primary-geo-node') }
- let(:secondary) { create(:geo_node) }
-
- before do
- stub_current_geo_node(secondary)
- allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true)
- allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:renew).and_return(true)
- allow_any_instance_of(described_class).to receive(:over_time?).and_return(false)
- WebMock.stub_request(:get, /primary-geo-node/).to_return(status: 200, body: "", headers: {})
- end
-
- subject { described_class.new }
-
- shared_examples '#perform' do |skip_tests|
- before do
- skip('FDW is not configured') if skip_tests
- end
-
- it 'does not schedule anything when tracking database is not configured' do
- create(:lfs_object, :with_file)
-
- allow(Gitlab::Geo).to receive(:geo_database_configured?) { false }
-
- expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
-
- subject.perform
-
- # We need to unstub here or the DatabaseCleaner will have issues since it
- # will appear as though the tracking DB were not available
- allow(Gitlab::Geo).to receive(:geo_database_configured?).and_call_original
- end
-
- it 'does not schedule anything when node is disabled' do
- create(:lfs_object, :with_file)
-
- secondary.enabled = false
- secondary.save
-
- expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
-
- subject.perform
- end
-
- context 'with LFS objects' do
- let!(:lfs_object_local_store) { create(:lfs_object, :with_file) }
- let!(:lfs_object_remote_store) { create(:lfs_object, :with_file) }
-
- before do
- stub_lfs_object_storage
- lfs_object_remote_store.file.migrate!(LfsObjectUploader::Store::REMOTE)
- end
-
- it 'filters S3-backed files' do
- expect(Geo::FileDownloadWorker).to receive(:perform_async).with(:lfs, lfs_object_local_store.id)
- expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with(:lfs, lfs_object_remote_store.id)
-
- subject.perform
- end
- end
-
- context 'with job artifacts' do
- it 'performs Geo::FileDownloadWorker for unsynced job artifacts' do
- artifact = create(:ci_job_artifact)
-
- expect(Geo::FileDownloadWorker).to receive(:perform_async)
- .with(:job_artifact, artifact.id).once.and_return(spy)
-
- subject.perform
- end
-
- it 'performs Geo::FileDownloadWorker for failed-sync job artifacts' do
- artifact = create(:ci_job_artifact)
-
- Geo::FileRegistry.create!(file_type: :job_artifact, file_id: artifact.id, bytes: 0, success: false)
-
- expect(Geo::FileDownloadWorker).to receive(:perform_async)
- .with('job_artifact', artifact.id).once.and_return(spy)
-
- subject.perform
- end
-
- it 'does not perform Geo::FileDownloadWorker for synced job artifacts' do
- artifact = create(:ci_job_artifact)
-
- Geo::FileRegistry.create!(file_type: :job_artifact, file_id: artifact.id, bytes: 1234, success: true)
-
- expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
-
- subject.perform
- end
-
- it 'does not perform Geo::FileDownloadWorker for synced job artifacts even with 0 bytes downloaded' do
- artifact = create(:ci_job_artifact)
-
- Geo::FileRegistry.create!(file_type: :job_artifact, file_id: artifact.id, bytes: 0, success: true)
-
- expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
-
- subject.perform
- end
- end
-
- # Test the case where we have:
- #
- # 1. A total of 10 files in the queue, and we can load a maximimum of 5 and send 2 at a time.
- # 2. We send 2, wait for 1 to finish, and then send again.
- it 'attempts to load a new batch without pending downloads' do
- stub_const('Geo::BaseSchedulerWorker::DB_RETRIEVE_BATCH_SIZE', 5)
- secondary.update!(files_max_capacity: 2)
- allow_any_instance_of(::Gitlab::Geo::Transfer).to receive(:download_from_primary).and_return(100)
-
- avatar = fixture_file_upload(Rails.root.join('spec/fixtures/dk.png'))
- create_list(:lfs_object, 2, :with_file)
- create_list(:user, 2, avatar: avatar)
- create_list(:note, 2, :with_attachment)
- create_list(:upload, 1, :personal_snippet_upload)
- create_list(:ci_job_artifact, 1)
- create(:appearance, logo: avatar, header_logo: avatar)
-
- expect(Geo::FileDownloadWorker).to receive(:perform_async).exactly(10).times.and_call_original
- # For 10 downloads, we expect four database reloads:
- # 1. Load the first batch of 5.
- # 2. 4 get sent out, 1 remains. This triggers another reload, which loads in the next 5.
- # 3. Those 4 get sent out, and 1 remains.
- # 3. Since the second reload filled the pipe with 4, we need to do a final reload to ensure
- # zero are left.
- expect(subject).to receive(:load_pending_resources).exactly(4).times.and_call_original
-
- Sidekiq::Testing.inline! do
- subject.perform
- end
- end
-
- context 'with a failed file' do
- let(:failed_registry) { create(:geo_file_registry, :lfs, file_id: 999, success: false) }
-
- it 'does not stall backfill' do
- unsynced = create(:lfs_object, :with_file)
-
- stub_const('Geo::BaseSchedulerWorker::DB_RETRIEVE_BATCH_SIZE', 1)
-
- expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with(:lfs, failed_registry.file_id)
- expect(Geo::FileDownloadWorker).to receive(:perform_async).with(:lfs, unsynced.id)
-
- subject.perform
- end
-
- it 'retries failed files' do
- expect(Geo::FileDownloadWorker).to receive(:perform_async).with('lfs', failed_registry.file_id)
-
- subject.perform
- end
-
- it 'does not retries failed files when retry_at is tomorrow' do
- failed_registry = create(:geo_file_registry, :lfs, file_id: 999, success: false, retry_at: Date.tomorrow)
-
- expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with('lfs', failed_registry.file_id)
-
- subject.perform
- end
-
- it 'does not retries failed files when retry_at is in the past' do
- failed_registry = create(:geo_file_registry, :lfs, file_id: 999, success: false, retry_at: Date.yesterday)
-
- expect(Geo::FileDownloadWorker).to receive(:perform_async).with('lfs', failed_registry.file_id)
-
- subject.perform
- end
- end
-
- context 'when node has namespace restrictions' do
- let(:synced_group) { create(:group) }
- let(:project_in_synced_group) { create(:project, group: synced_group) }
- let(:unsynced_project) { create(:project) }
-
- before do
- allow(ProjectCacheWorker).to receive(:perform_async).and_return(true)
-
- secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
- end
-
- it 'does not perform Geo::FileDownloadWorker for LFS object that does not belong to selected namespaces to replicate' do
- lfs_object_in_synced_group = create(:lfs_objects_project, project: project_in_synced_group)
- create(:lfs_objects_project, project: unsynced_project)
-
- expect(Geo::FileDownloadWorker).to receive(:perform_async)
- .with(:lfs, lfs_object_in_synced_group.lfs_object_id).once.and_return(spy)
-
- subject.perform
- end
-
- it 'does not perform Geo::FileDownloadWorker for job artifact that does not belong to selected namespaces to replicate' do
- create(:ci_job_artifact, project: unsynced_project)
- job_artifact_in_synced_group = create(:ci_job_artifact, project: project_in_synced_group)
-
- expect(Geo::FileDownloadWorker).to receive(:perform_async)
- .with(:job_artifact, job_artifact_in_synced_group.id).once.and_return(spy)
-
- subject.perform
- end
-
- it 'does not perform Geo::FileDownloadWorker for upload objects that do not belong to selected namespaces to replicate' do
- avatar = fixture_file_upload(Rails.root.join('spec/fixtures/dk.png'))
- avatar_in_synced_group = create(:upload, model: synced_group, path: avatar)
- create(:upload, model: create(:group), path: avatar)
- avatar_in_project_in_synced_group = create(:upload, model: project_in_synced_group, path: avatar)
- create(:upload, model: unsynced_project, path: avatar)
-
- expect(Geo::FileDownloadWorker).to receive(:perform_async)
- .with('avatar', avatar_in_project_in_synced_group.id).once.and_return(spy)
-
- expect(Geo::FileDownloadWorker).to receive(:perform_async)
- .with('avatar', avatar_in_synced_group.id).once.and_return(spy)
-
- subject.perform
- end
- end
- end
-
- # Disable transactions via :delete method because a foreign table
- # can't see changes inside a transaction of a different connection.
- describe 'when PostgreSQL FDW is available', :geo, :delete do
- # Skip if FDW isn't activated on this database
- it_behaves_like '#perform', Gitlab::Database.postgresql? && !Gitlab::Geo.fdw?
- end
-
- describe 'when PostgreSQL FDW is not enabled', :geo do
- before do
- allow(Gitlab::Geo).to receive(:fdw?).and_return(false)
- end
-
- it_behaves_like '#perform', false
- end
-
- describe '#take_batch' do
- it 'returns a batch of jobs' do
- a = [[2, :lfs], [3, :lfs]]
- b = []
- c = [[3, :job_artifact], [8, :job_artifact], [9, :job_artifact]]
- expect(subject).to receive(:db_retrieve_batch_size).and_return(4)
-
- expect(subject.send(:take_batch, a, b, c)).to eq([
- [3, :job_artifact],
- [2, :lfs],
- [8, :job_artifact],
- [3, :lfs]
- ])
- end
- end
-
- describe '#interleave' do
- # Notice ties are resolved by taking the "first" tied element
- it 'interleaves 2 arrays' do
- a = %w{1 2 3}
- b = %w{A B C}
- expect(subject.send(:interleave, a, b)).to eq(%w{1 A 2 B 3 C})
- end
-
- # Notice there are no ties in this call
- it 'interleaves 2 arrays with a longer second array' do
- a = %w{1 2}
- b = %w{A B C}
- expect(subject.send(:interleave, a, b)).to eq(%w{A 1 B 2 C})
- end
-
- it 'interleaves 2 arrays with a longer first array' do
- a = %w{1 2 3}
- b = %w{A B}
- expect(subject.send(:interleave, a, b)).to eq(%w{1 A 2 B 3})
- end
-
- it 'interleaves 3 arrays' do
- a = %w{1 2 3}
- b = %w{A B C}
- c = %w{i ii iii}
- expect(subject.send(:interleave, a, b, c)).to eq(%w{1 A i 2 B ii 3 C iii})
- end
-
- it 'interleaves 3 arrays of unequal length' do
- a = %w{1 2}
- b = %w{A}
- c = %w{i ii iii iiii}
- expect(subject.send(:interleave, a, b, c)).to eq(%w{i 1 ii A iii 2 iiii})
- end
- end
-end
diff --git a/spec/factories/geo/event_log.rb b/spec/factories/geo/event_log.rb
deleted file mode 100644
index dbe2f400f97..00000000000
--- a/spec/factories/geo/event_log.rb
+++ /dev/null
@@ -1,121 +0,0 @@
-FactoryBot.define do
- factory :geo_event_log, class: Geo::EventLog do
- trait :created_event do
- repository_created_event factory: :geo_repository_created_event
- end
-
- trait :updated_event do
- repository_updated_event factory: :geo_repository_updated_event
- end
-
- trait :deleted_event do
- repository_deleted_event factory: :geo_repository_deleted_event
- end
-
- trait :renamed_event do
- repository_renamed_event factory: :geo_repository_renamed_event
- end
-
- trait :hashed_storage_migration_event do
- hashed_storage_migrated_event factory: :geo_hashed_storage_migrated_event
- end
-
- trait :hashed_storage_attachments_event do
- hashed_storage_attachments_event factory: :geo_hashed_storage_attachments_event
- end
-
- trait :lfs_object_deleted_event do
- lfs_object_deleted_event factory: :geo_lfs_object_deleted_event
- end
-
- trait :job_artifact_deleted_event do
- job_artifact_deleted_event factory: :geo_job_artifact_deleted_event
- end
- end
-
- factory :geo_repository_created_event, class: Geo::RepositoryCreatedEvent do
- project
-
- repository_storage_name { project.repository_storage }
- repository_storage_path { project.repository_storage_path }
- add_attribute(:repo_path) { project.disk_path }
- project_name { project.name }
- wiki_path { project.wiki.disk_path }
- end
-
- factory :geo_repository_updated_event, class: Geo::RepositoryUpdatedEvent do
- project
-
- source 0
- branches_affected 0
- tags_affected 0
- end
-
- factory :geo_repository_deleted_event, class: Geo::RepositoryDeletedEvent do
- project
-
- repository_storage_name { project.repository_storage }
- repository_storage_path { project.repository_storage_path }
- deleted_path { project.path_with_namespace }
- deleted_project_name { project.name }
- end
-
- factory :geo_repositories_changed_event, class: Geo::RepositoriesChangedEvent do
- geo_node
- end
-
- factory :geo_repository_renamed_event, class: Geo::RepositoryRenamedEvent do
- project { create(:project, :repository) }
-
- repository_storage_name { project.repository_storage }
- repository_storage_path { project.repository_storage_path }
- old_path_with_namespace { project.path_with_namespace }
- new_path_with_namespace { project.path_with_namespace + '_new' }
- old_wiki_path_with_namespace { project.wiki.path_with_namespace }
- new_wiki_path_with_namespace { project.wiki.path_with_namespace + '_new' }
- old_path { project.path }
- new_path { project.path + '_new' }
- end
-
- factory :geo_hashed_storage_migrated_event, class: Geo::HashedStorageMigratedEvent do
- project { create(:project, :repository) }
-
- repository_storage_name { project.repository_storage }
- repository_storage_path { project.repository_storage_path }
- old_disk_path { project.path_with_namespace }
- new_disk_path { project.path_with_namespace + '_new' }
- old_wiki_disk_path { project.wiki.path_with_namespace }
- new_wiki_disk_path { project.wiki.path_with_namespace + '_new' }
- new_storage_version { Project::HASHED_STORAGE_FEATURES[:repository] }
- end
-
- factory :geo_hashed_storage_attachments_event, class: Geo::HashedStorageAttachmentsEvent do
- project { create(:project, :repository) }
-
- old_attachments_path { Storage::LegacyProject.new(project).disk_path }
- new_attachments_path { Storage::HashedProject.new(project).disk_path }
- end
-
- factory :geo_lfs_object_deleted_event, class: Geo::LfsObjectDeletedEvent do
- lfs_object { create(:lfs_object, :with_file) }
-
- after(:build, :stub) do |event, _|
- local_store_path = Pathname.new(LfsObjectUploader.root)
- relative_path = Pathname.new(event.lfs_object.file.path).relative_path_from(local_store_path)
-
- event.oid = event.lfs_object.oid
- event.file_path = relative_path
- end
- end
-
- factory :geo_job_artifact_deleted_event, class: Geo::JobArtifactDeletedEvent do
- job_artifact { create(:ci_job_artifact, :archive) }
-
- after(:build, :stub) do |event, _|
- local_store_path = Pathname.new(JobArtifactUploader.root)
- relative_path = Pathname.new(event.job_artifact.file.path).relative_path_from(local_store_path)
-
- event.file_path = relative_path
- end
- end
-end
diff --git a/spec/factories/uploads.rb b/spec/factories/uploads.rb
index 008d9616216..b45f6f30e40 100644
--- a/spec/factories/uploads.rb
+++ b/spec/factories/uploads.rb
@@ -3,7 +3,6 @@ FactoryBot.define do
model { build(:project) }
size 100.kilobytes
uploader "AvatarUploader"
- store ObjectStorage::Store::LOCAL
mount_point :avatar
secret nil
store ObjectStorage::Store::LOCAL
diff --git a/spec/models/ci/job_artifact_spec.rb b/spec/models/ci/job_artifact_spec.rb
index 1e8860cf295..a244c2c1762 100644
--- a/spec/models/ci/job_artifact_spec.rb
+++ b/spec/models/ci/job_artifact_spec.rb
@@ -18,7 +18,7 @@ describe Ci::JobArtifact do
describe 'callbacks' do
subject { create(:ci_job_artifact, :archive) }
- describe '#schedule_migration_to_object_storage' do
+ describe '#schedule_background_upload' do
context 'when object storage is disabled' do
before do
stub_artifacts_object_storage(enabled: false)
diff --git a/spec/support/shared_examples/uploaders/gitlab_uploader_shared_examples.rb b/spec/support/shared_examples/uploaders/gitlab_uploader_shared_examples.rb
new file mode 100644
index 00000000000..934d53e7bba
--- /dev/null
+++ b/spec/support/shared_examples/uploaders/gitlab_uploader_shared_examples.rb
@@ -0,0 +1,48 @@
+shared_examples "matches the method pattern" do |method|
+ let(:target) { subject }
+ let(:args) { nil }
+ let(:pattern) { patterns[method] }
+
+ it do
+ return skip "No pattern provided, skipping." unless pattern
+
+ expect(target.method(method).call(*args)).to match(pattern)
+ end
+end
+
+shared_examples "builds correct paths" do |**patterns|
+ let(:patterns) { patterns }
+
+ before do
+ allow(subject).to receive(:filename).and_return('<filename>')
+ end
+
+ describe "#store_dir" do
+ it_behaves_like "matches the method pattern", :store_dir
+ end
+
+ describe "#cache_dir" do
+ it_behaves_like "matches the method pattern", :cache_dir
+ end
+
+ describe "#work_dir" do
+ it_behaves_like "matches the method pattern", :work_dir
+ end
+
+ describe "#upload_path" do
+ it_behaves_like "matches the method pattern", :upload_path
+ end
+
+ describe ".absolute_path" do
+ it_behaves_like "matches the method pattern", :absolute_path do
+ let(:target) { subject.class }
+ let(:args) { [upload] }
+ end
+ end
+
+ describe ".base_dir" do
+ it_behaves_like "matches the method pattern", :base_dir do
+ let(:target) { subject.class }
+ end
+ end
+end
diff --git a/spec/ee/workers/object_storage_upload_worker_spec.rb b/spec/workers/object_storage_upload_worker_spec.rb
index 32ddcbe9757..5edc4bc499b 100644
--- a/spec/ee/workers/object_storage_upload_worker_spec.rb
+++ b/spec/workers/object_storage_upload_worker_spec.rb
@@ -20,7 +20,7 @@ describe ObjectStorageUploadWorker do
stub_lfs_object_storage(background_upload: true)
end
- it 'uploads object to storage' do
+ it 'uploads object to storage' d
expect { perform }.to change { lfs_object.reload.file_store }.from(local).to(remote)
end