Merge branch '4163-move-uploads-to-object-storage' into 'master'

Move uploads to object storage

Closes #4163

See merge request gitlab-org/gitlab-ee!3867

7 files changed, 150 insertions, 10 deletions

diff --git a/spec/factories/ci/job_artifacts.rb b/spec/factories/ci/job_artifacts.rb
index 436735e7ed3..9bb456e89ff 100644
--- a/spec/factories/ci/job_artifacts.rb
+++ b/spec/factories/ci/job_artifacts.rb
@@ -6,7 +6,7 @@ FactoryBot.define do
     file_type :archive
 
     trait :remote_store do
-      file_store JobArtifactUploader::REMOTE_STORE
+      file_store JobArtifactUploader::Store::REMOTE
     end
 
     after :build do |artifact|
diff --git a/spec/factories/geo/event_log.rb b/spec/factories/geo/event_log.rb
new file mode 100644
index 00000000000..dbe2f400f97
--- /dev/null
+++ b/spec/factories/geo/event_log.rb
@@ -0,0 +1,121 @@
+FactoryBot.define do
+  factory :geo_event_log, class: Geo::EventLog do
+    trait :created_event do
+      repository_created_event factory: :geo_repository_created_event
+    end
+
+    trait :updated_event do
+      repository_updated_event factory: :geo_repository_updated_event
+    end
+
+    trait :deleted_event do
+      repository_deleted_event factory: :geo_repository_deleted_event
+    end
+
+    trait :renamed_event do
+      repository_renamed_event factory: :geo_repository_renamed_event
+    end
+
+    trait :hashed_storage_migration_event do
+      hashed_storage_migrated_event factory: :geo_hashed_storage_migrated_event
+    end
+
+    trait :hashed_storage_attachments_event do
+      hashed_storage_attachments_event factory: :geo_hashed_storage_attachments_event
+    end
+
+    trait :lfs_object_deleted_event do
+      lfs_object_deleted_event factory: :geo_lfs_object_deleted_event
+    end
+
+    trait :job_artifact_deleted_event do
+      job_artifact_deleted_event factory: :geo_job_artifact_deleted_event
+    end
+  end
+
+  factory :geo_repository_created_event, class: Geo::RepositoryCreatedEvent do
+    project
+
+    repository_storage_name { project.repository_storage }
+    repository_storage_path { project.repository_storage_path }
+    add_attribute(:repo_path) { project.disk_path }
+    project_name { project.name }
+    wiki_path { project.wiki.disk_path }
+  end
+
+  factory :geo_repository_updated_event, class: Geo::RepositoryUpdatedEvent do
+    project
+
+    source 0
+    branches_affected 0
+    tags_affected 0
+  end
+
+  factory :geo_repository_deleted_event, class: Geo::RepositoryDeletedEvent do
+    project
+
+    repository_storage_name { project.repository_storage }
+    repository_storage_path { project.repository_storage_path }
+    deleted_path { project.path_with_namespace }
+    deleted_project_name { project.name }
+  end
+
+  factory :geo_repositories_changed_event, class: Geo::RepositoriesChangedEvent do
+    geo_node
+  end
+
+  factory :geo_repository_renamed_event, class: Geo::RepositoryRenamedEvent do
+    project { create(:project, :repository) }
+
+    repository_storage_name { project.repository_storage }
+    repository_storage_path { project.repository_storage_path }
+    old_path_with_namespace { project.path_with_namespace }
+    new_path_with_namespace { project.path_with_namespace + '_new' }
+    old_wiki_path_with_namespace { project.wiki.path_with_namespace }
+    new_wiki_path_with_namespace { project.wiki.path_with_namespace + '_new' }
+    old_path { project.path }
+    new_path { project.path + '_new' }
+  end
+
+  factory :geo_hashed_storage_migrated_event, class: Geo::HashedStorageMigratedEvent do
+    project { create(:project, :repository) }
+
+    repository_storage_name { project.repository_storage }
+    repository_storage_path { project.repository_storage_path }
+    old_disk_path { project.path_with_namespace }
+    new_disk_path { project.path_with_namespace + '_new' }
+    old_wiki_disk_path { project.wiki.path_with_namespace }
+    new_wiki_disk_path { project.wiki.path_with_namespace + '_new' }
+    new_storage_version { Project::HASHED_STORAGE_FEATURES[:repository] }
+  end
+
+  factory :geo_hashed_storage_attachments_event, class: Geo::HashedStorageAttachmentsEvent do
+    project { create(:project, :repository) }
+
+    old_attachments_path { Storage::LegacyProject.new(project).disk_path }
+    new_attachments_path { Storage::HashedProject.new(project).disk_path }
+  end
+
+  factory :geo_lfs_object_deleted_event, class: Geo::LfsObjectDeletedEvent do
+    lfs_object { create(:lfs_object, :with_file) }
+
+    after(:build, :stub) do |event, _|
+      local_store_path = Pathname.new(LfsObjectUploader.root)
+      relative_path = Pathname.new(event.lfs_object.file.path).relative_path_from(local_store_path)
+
+      event.oid = event.lfs_object.oid
+      event.file_path = relative_path
+    end
+  end
+
+  factory :geo_job_artifact_deleted_event, class: Geo::JobArtifactDeletedEvent do
+    job_artifact { create(:ci_job_artifact, :archive) }
+
+    after(:build, :stub) do |event, _|
+      local_store_path = Pathname.new(JobArtifactUploader.root)
+      relative_path = Pathname.new(event.job_artifact.file.path).relative_path_from(local_store_path)
+
+      event.file_path = relative_path
+    end
+  end
+end
diff --git a/spec/factories/groups.rb b/spec/factories/groups.rb
index 1512f5a0e58..8c531cf5909 100644
--- a/spec/factories/groups.rb
+++ b/spec/factories/groups.rb
@@ -18,7 +18,7 @@ FactoryBot.define do
     end
 
     trait :with_avatar do
-      avatar { File.open(Rails.root.join('spec/fixtures/dk.png')) }
+      avatar { fixture_file_upload('spec/fixtures/dk.png') }
     end
 
     trait :access_requestable do
diff --git a/spec/factories/notes.rb b/spec/factories/notes.rb
index 707ecbd6be5..0889c5090fb 100644
--- a/spec/factories/notes.rb
+++ b/spec/factories/notes.rb
@@ -122,11 +122,11 @@ FactoryBot.define do
     end
 
     trait :with_attachment do
-      attachment { fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "image/png") }
+      attachment { fixture_file_upload(Rails.root.join( "spec/fixtures/dk.png"), "image/png") }
     end
 
     trait :with_svg_attachment do
-      attachment { fixture_file_upload(Rails.root + "spec/fixtures/unsanitized.svg", "image/svg+xml") }
+      attachment { fixture_file_upload(Rails.root.join("spec/fixtures/unsanitized.svg"), "image/svg+xml") }
     end
 
     transient do
diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb
index d0f3911f730..16d328a5bc2 100644
--- a/spec/factories/projects.rb
+++ b/spec/factories/projects.rb
@@ -90,7 +90,7 @@ FactoryBot.define do
     end
 
     trait :with_avatar do
-      avatar { File.open(Rails.root.join('spec/fixtures/dk.png')) }
+      avatar { fixture_file_upload('spec/fixtures/dk.png') }
     end
 
     trait :broken_storage do
diff --git a/spec/factories/uploads.rb b/spec/factories/uploads.rb
index c39500faea1..9e8a55eaedb 100644
--- a/spec/factories/uploads.rb
+++ b/spec/factories/uploads.rb
@@ -1,24 +1,43 @@
 FactoryBot.define do
   factory :upload do
     model { build(:project) }
-    path { "uploads/-/system/project/avatar/avatar.jpg" }
     size 100.kilobytes
     uploader "AvatarUploader"
+    store ObjectStorage::Store::LOCAL
 
-    trait :personal_snippet do
+    # we should build a mount agnostic upload by default
+    transient do
+      mounted_as :avatar
+      secret SecureRandom.hex
+    end
+
+    # this needs to comply with RecordsUpload::Concern#upload_path
+    path { File.join("uploads/-/system", model.class.to_s.underscore, mounted_as.to_s, 'avatar.jpg') }
+
+    trait :personal_snippet_upload do
       model { build(:personal_snippet) }
+      path { File.join(secret, 'myfile.jpg') }
       uploader "PersonalFileUploader"
     end
 
     trait :issuable_upload do
-      path { "#{SecureRandom.hex}/myfile.jpg" }
+      path { File.join(secret, 'myfile.jpg') }
       uploader "FileUploader"
     end
 
     trait :namespace_upload do
-      path { "#{SecureRandom.hex}/myfile.jpg" }
       model { build(:group) }
+      path { File.join(secret, 'myfile.jpg') }
       uploader "NamespaceFileUploader"
     end
+
+    trait :attachment_upload do
+      transient do
+        mounted_as :attachment
+      end
+
+      model { build(:note) }
+      uploader "AttachmentUploader"
+    end
   end
 end
diff --git a/spec/factories/users.rb b/spec/factories/users.rb
index e62e0b263ca..769fd656e7a 100644
--- a/spec/factories/users.rb
+++ b/spec/factories/users.rb
@@ -38,7 +38,7 @@ FactoryBot.define do
     end
 
     trait :with_avatar do
-      avatar { File.open(Rails.root.join('spec/fixtures/dk.png')) }
+      avatar { fixture_file_upload('spec/fixtures/dk.png') }
     end
 
     trait :two_factor_via_otp do