Merge branch '40781-os-to-ce' into 'master'

Bring Object Storage to CE

Closes #4171, #4163, #3370, #2841, and #29203

See merge request gitlab-org/gitlab-ce!17358

3 files changed, 233 insertions, 0 deletions

diff --git a/spec/support/http_io/http_io_helpers.rb b/spec/support/http_io/http_io_helpers.rb
new file mode 100644
index 00000000000..31e07e720cd
--- /dev/null
+++ b/spec/support/http_io/http_io_helpers.rb
@@ -0,0 +1,64 @@
+module HttpIOHelpers
+  def stub_remote_trace_206
+    WebMock.stub_request(:get, remote_trace_url)
+      .to_return { |request| remote_trace_response(request, 206) }
+  end
+
+  def stub_remote_trace_200
+    WebMock.stub_request(:get, remote_trace_url)
+      .to_return { |request| remote_trace_response(request, 200) }
+  end
+
+  def stub_remote_trace_500
+    WebMock.stub_request(:get, remote_trace_url)
+      .to_return(status: [500, "Internal Server Error"])
+  end
+
+  def remote_trace_url
+    "http://trace.com/trace"
+  end
+
+  def remote_trace_response(request, responce_status)
+    range = request.headers['Range'].match(/bytes=(\d+)-(\d+)/)
+
+    {
+      status: responce_status,
+      headers: remote_trace_response_headers(responce_status, range[1].to_i, range[2].to_i),
+      body: range_trace_body(range[1].to_i, range[2].to_i)
+    }
+  end
+
+  def remote_trace_response_headers(responce_status, from, to)
+    headers = { 'Content-Type' => 'text/plain' }
+
+    if responce_status == 206
+      headers.merge('Content-Range' => "bytes #{from}-#{to}/#{remote_trace_size}")
+    end
+
+    headers
+  end
+
+  def range_trace_body(from, to)
+    remote_trace_body[from..to]
+  end
+
+  def remote_trace_body
+    @remote_trace_body ||= File.read(expand_fixture_path('trace/sample_trace'))
+  end
+
+  def remote_trace_size
+    remote_trace_body.length
+  end
+
+  def set_smaller_buffer_size_than(file_size)
+    blocks = (file_size / 128)
+    new_size = (blocks / 2) * 128
+    stub_const("Gitlab::Ci::Trace::HttpIO::BUFFER_SIZE", new_size)
+  end
+
+  def set_larger_buffer_size_than(file_size)
+    blocks = (file_size / 128)
+    new_size = (blocks * 2) * 128
+    stub_const("Gitlab::Ci::Trace::HttpIO::BUFFER_SIZE", new_size)
+  end
+end
diff --git a/spec/support/shared_examples/uploaders/object_storage_shared_examples.rb b/spec/support/shared_examples/uploaders/object_storage_shared_examples.rb
new file mode 100644
index 00000000000..cd9974cd6e2
--- /dev/null
+++ b/spec/support/shared_examples/uploaders/object_storage_shared_examples.rb
@@ -0,0 +1,126 @@
+shared_context 'with storage' do |store, **stub_params|
+  before do
+    subject.object_store = store
+  end
+end
+
+shared_examples "migrates" do |to_store:, from_store: nil|
+  let(:to) { to_store }
+  let(:from) { from_store || subject.object_store }
+
+  def migrate(to)
+    subject.migrate!(to)
+  end
+
+  def checksum
+    Digest::SHA256.hexdigest(subject.read)
+  end
+
+  before do
+    migrate(from)
+  end
+
+  it 'returns corresponding file type' do
+    expect(subject).to be_an(CarrierWave::Uploader::Base)
+    expect(subject).to be_a(ObjectStorage::Concern)
+
+    if from == described_class::Store::REMOTE
+      expect(subject.file).to be_a(CarrierWave::Storage::Fog::File)
+    elsif from == described_class::Store::LOCAL
+      expect(subject.file).to be_a(CarrierWave::SanitizedFile)
+    else
+      raise 'Unexpected file type'
+    end
+  end
+
+  it 'does nothing when migrating to the current store' do
+    expect { migrate(from) }.not_to change { subject.object_store }.from(from)
+  end
+
+  it 'migrate to the specified store' do
+    from_checksum = checksum
+
+    expect { migrate(to) }.to change { subject.object_store }.from(from).to(to)
+    expect(checksum).to eq(from_checksum)
+  end
+
+  it 'removes the original file after the migration' do
+    original_file = subject.file.path
+    migrate(to)
+
+    expect(File.exist?(original_file)).to be_falsey
+  end
+
+  it 'can access to the original file during migration' do
+    file = subject.file
+
+    allow(subject).to receive(:delete_migrated_file) { } # Remove as a callback of :migrate
+    allow(subject).to receive(:record_upload) { } # Remove as a callback of :store (:record_upload)
+
+    expect(file.exists?).to be_truthy
+    expect { migrate(to) }.not_to change { file.exists? }
+  end
+
+  context 'when migrate! is not oqqupied by another process' do
+    it 'executes migrate!' do
+      expect(subject).to receive(:object_store=).at_least(1)
+
+      migrate(to)
+    end
+  end
+
+  context 'when migrate! is occupied by another process' do
+    let(:exclusive_lease_key) { "object_storage_migrate:#{subject.model.class}:#{subject.model.id}" }
+
+    before do
+      @uuid = Gitlab::ExclusiveLease.new(exclusive_lease_key, timeout: 1.hour.to_i).try_obtain
+    end
+
+    it 'does not execute migrate!' do
+      expect(subject).not_to receive(:unsafe_migrate!)
+
+      expect { migrate(to) }.to raise_error('Already running')
+    end
+
+    after do
+      Gitlab::ExclusiveLease.cancel(exclusive_lease_key, @uuid)
+    end
+  end
+
+  context 'migration is unsuccessful' do
+    shared_examples "handles gracefully" do |error:|
+      it 'does not update the object_store' do
+        expect { migrate(to) }.to raise_error(error)
+        expect(subject.object_store).to eq(from)
+      end
+
+      it 'does not delete the original file' do
+        expect { migrate(to) }.to raise_error(error)
+        expect(subject.exists?).to be_truthy
+      end
+    end
+
+    context 'when the store is not supported' do
+      let(:to) { -1 } # not a valid store
+
+      include_examples "handles gracefully", error: ObjectStorage::UnknownStoreError
+    end
+
+    context 'upon a fog failure' do
+      before do
+        storage_class = subject.send(:storage_for, to).class
+        expect_any_instance_of(storage_class).to receive(:store!).and_raise("Store failure.")
+      end
+
+      include_examples "handles gracefully", error: "Store failure."
+    end
+
+    context 'upon a database failure' do
+      before do
+        expect(uploader).to receive(:persist_object_store!).and_raise("ActiveRecord failure.")
+      end
+
+      include_examples "handles gracefully", error: "ActiveRecord failure."
+    end
+  end
+end
diff --git a/spec/support/stub_object_storage.rb b/spec/support/stub_object_storage.rb
new file mode 100644
index 00000000000..1a0a2feb27d
--- /dev/null
+++ b/spec/support/stub_object_storage.rb
@@ -0,0 +1,43 @@
+module StubConfiguration
+  def stub_object_storage_uploader(
+        config:, uploader:, remote_directory:,
+        enabled: true,
+        proxy_download: false,
+        background_upload: false)
+    Fog.mock!
+
+    allow(config).to receive(:enabled) { enabled }
+    allow(config).to receive(:proxy_download) { proxy_download }
+    allow(config).to receive(:background_upload) { background_upload }
+
+    return unless enabled
+
+    ::Fog::Storage.new(uploader.object_store_credentials).tap do |connection|
+      begin
+        connection.directories.create(key: remote_directory)
+      rescue Excon::Error::Conflict
+      end
+    end
+  end
+
+  def stub_artifacts_object_storage(**params)
+    stub_object_storage_uploader(config: Gitlab.config.artifacts.object_store,
+                                 uploader: JobArtifactUploader,
+                                 remote_directory: 'artifacts',
+                                 **params)
+  end
+
+  def stub_lfs_object_storage(**params)
+    stub_object_storage_uploader(config: Gitlab.config.lfs.object_store,
+                                 uploader: LfsObjectUploader,
+                                 remote_directory: 'lfs-objects',
+                                 **params)
+  end
+
+  def stub_uploads_object_storage(uploader = described_class, **params)
+    stub_object_storage_uploader(config: Gitlab.config.uploads.object_store,
+                                 uploader: uploader,
+                                 remote_directory: 'uploads',
+                                 **params)
+  end
+end