Add latest changes from gitlab-org/gitlab@13-5-stable-ee

3 files changed, 104 insertions, 3 deletions

diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb
index 6926ccd9438..cf6eb159f52 100644
--- a/app/models/ci/build_trace_chunk.rb
+++ b/app/models/ci/build_trace_chunk.rb
@@ -45,6 +45,10 @@ module Ci
 
       def get_store_class(store)
         @stores ||= {}
+
+        # Can't memoize this because the feature flag may alter this
+        return fog_store_class.new if store.to_sym == :fog
+
         @stores[store] ||= "Ci::BuildTraceChunks::#{store.capitalize}".constantize.new
       end
 
@@ -74,6 +78,14 @@ module Ci
       def metadata_attributes
         attribute_names - %w[raw_data]
       end
+
+      def fog_store_class
+        if Feature.enabled?(:ci_trace_new_fog_store, default_enabled: true)
+          Ci::BuildTraceChunks::Fog
+        else
+          Ci::BuildTraceChunks::LegacyFog
+        end
+      end
     end
 
     def data
diff --git a/app/models/ci/build_trace_chunks/fog.rb b/app/models/ci/build_trace_chunks/fog.rb
index b1e9fd1faeb..d3051e3dadc 100644
--- a/app/models/ci/build_trace_chunks/fog.rb
+++ b/app/models/ci/build_trace_chunks/fog.rb
@@ -8,13 +8,17 @@ module Ci
       end
 
       def data(model)
-        connection.get_object(bucket_name, key(model))[:body]
+        files.get(key(model))&.body
       rescue Excon::Error::NotFound
         # If the object does not exist in the object storage, this method returns nil.
       end
 
       def set_data(model, new_data)
-        connection.put_object(bucket_name, key(model), new_data)
+        # TODO: Support AWS S3 server side encryption
+        files.create({
+          key: key(model),
+          body: new_data
+        })
       end
 
       def append_data(model, new_data, offset)
@@ -43,7 +47,7 @@ module Ci
 
       def delete_keys(keys)
         keys.each do |key|
-          connection.delete_object(bucket_name, key_raw(*key))
+          files.destroy(key_raw(*key))
         end
       end
 
@@ -69,6 +73,14 @@ module Ci
         @connection ||= ::Fog::Storage.new(object_store.connection.to_hash.deep_symbolize_keys)
       end
 
+      def fog_directory
+        @fog_directory ||= connection.directories.new(key: bucket_name)
+      end
+
+      def files
+        @files ||= fog_directory.files
+      end
+
       def object_store
         Gitlab.config.artifacts.object_store
       end
diff --git a/app/models/ci/build_trace_chunks/legacy_fog.rb b/app/models/ci/build_trace_chunks/legacy_fog.rb
new file mode 100644
index 00000000000..b710ed2890b
--- /dev/null
+++ b/app/models/ci/build_trace_chunks/legacy_fog.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Ci
+  module BuildTraceChunks
+    class LegacyFog
+      def available?
+        object_store.enabled
+      end
+
+      def data(model)
+        connection.get_object(bucket_name, key(model))[:body]
+      rescue Excon::Error::NotFound
+        # If the object does not exist in the object storage, this method returns nil.
+      end
+
+      def set_data(model, new_data)
+        connection.put_object(bucket_name, key(model), new_data)
+      end
+
+      def append_data(model, new_data, offset)
+        if offset > 0
+          truncated_data = data(model).to_s.byteslice(0, offset)
+          new_data = truncated_data + new_data
+        end
+
+        set_data(model, new_data)
+        new_data.bytesize
+      end
+
+      def size(model)
+        data(model).to_s.bytesize
+      end
+
+      def delete_data(model)
+        delete_keys([[model.build_id, model.chunk_index]])
+      end
+
+      def keys(relation)
+        return [] unless available?
+
+        relation.pluck(:build_id, :chunk_index)
+      end
+
+      def delete_keys(keys)
+        keys.each do |key|
+          connection.delete_object(bucket_name, key_raw(*key))
+        end
+      end
+
+      private
+
+      def key(model)
+        key_raw(model.build_id, model.chunk_index)
+      end
+
+      def key_raw(build_id, chunk_index)
+        "tmp/builds/#{build_id.to_i}/chunks/#{chunk_index.to_i}.log"
+      end
+
+      def bucket_name
+        return unless available?
+
+        object_store.remote_directory
+      end
+
+      def connection
+        return unless available?
+
+        @connection ||= ::Fog::Storage.new(object_store.connection.to_hash.deep_symbolize_keys)
+      end
+
+      def object_store
+        Gitlab.config.artifacts.object_store
+      end
+    end
+  end
+end