Very RAW draft of changes needed to migrate storage to MinIOminio-poc-dr

14 files changed, 58 insertions, 25 deletions

diff --git a/Gemfile b/Gemfile
index 9e815925a1f..6387831a204 100644
--- a/Gemfile
+++ b/Gemfile
@@ -78,7 +78,8 @@ gem 'kaminari', '~> 0.17.0'
 gem 'hamlit', '~> 2.6.1'
 
 # Files attachments
-gem 'carrierwave', '~> 0.10.0'
+gem 'carrierwave', '~> 0.11.0'
+gem 'carrierwave-aws', '~> 1.0'
 
 # Drag and Drop UI
 gem 'dropzonejs-rails', '~> 0.7.1'
diff --git a/Gemfile.lock b/Gemfile.lock
index bdc60552480..472cee57e09 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -62,6 +62,12 @@ GEM
       execjs
       json
     awesome_print (1.2.0)
+    aws-sdk (2.6.12)
+      aws-sdk-resources (= 2.6.12)
+    aws-sdk-core (2.6.12)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.6.12)
+      aws-sdk-core (= 2.6.12)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
@@ -117,11 +123,15 @@ GEM
     capybara-screenshot (1.0.11)
       capybara (>= 1.0, < 3)
       launchy
-    carrierwave (0.10.0)
+    carrierwave (0.11.2)
       activemodel (>= 3.2.0)
       activesupport (>= 3.2.0)
       json (>= 1.7)
       mime-types (>= 1.16)
+      mimemagic (>= 0.3.0)
+    carrierwave-aws (1.0.2)
+      aws-sdk (~> 2.0)
+      carrierwave (>= 0.7, < 2.0)
     cause (0.1)
     charlock_holmes (0.7.3)
     chronic (0.10.2)
@@ -355,6 +365,7 @@ GEM
     jira-ruby (1.1.2)
       activesupport
       oauth (~> 0.5, >= 0.5.0)
+    jmespath (1.3.1)
     jquery-atwho-rails (1.3.2)
     jquery-rails (4.1.1)
       rails-dom-testing (>= 1, < 3)
@@ -827,7 +838,8 @@ DEPENDENCIES
   byebug (~> 8.2.1)
   capybara (~> 2.6.2)
   capybara-screenshot (~> 1.0.0)
-  carrierwave (~> 0.10.0)
+  carrierwave (~> 0.11.0)
+  carrierwave-aws (~> 1.0)
   charlock_holmes (~> 0.7.3)
   chronic (~> 0.10.2)
   chronic_duration (~> 0.10.6)
diff --git a/app/controllers/projects/lfs_storage_controller.rb b/app/controllers/projects/lfs_storage_controller.rb
index 9005b104e90..209c6935ab6 100644
--- a/app/controllers/projects/lfs_storage_controller.rb
+++ b/app/controllers/projects/lfs_storage_controller.rb
@@ -12,7 +12,9 @@ class Projects::LfsStorageController < Projects::GitHttpClientController
       return
     end
 
-    send_file lfs_object.file.path, content_type: "application/octet-stream"
+    #send_file lfs_object.file.path, content_type: "application/octet-stream"
+    data = open(lfs_object.file.url)
+    send_data data.read, content_type: "application/octet-stream"
   end
 
   def upload_authorize
diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb
index 10d24da16d7..ef747de0004 100644
--- a/app/controllers/projects/raw_controller.rb
+++ b/app/controllers/projects/raw_controller.rb
@@ -31,7 +31,9 @@ class Projects::RawController < Projects::ApplicationController
     lfs_object = find_lfs_object
 
     if lfs_object && lfs_object.project_allowed_access?(@project)
-      send_file lfs_object.file.path, filename: @blob.name, disposition: 'attachment'
+      # send_file lfs_object.file.path, filename: @blob.name, disposition: 'attachment'
+      data = open(lfs_object.file.url)
+      send_data data.read, filename: @blob.name, disposition: 'attachment'
     else
       render_404
     end
diff --git a/app/controllers/projects/uploads_controller.rb b/app/controllers/projects/uploads_controller.rb
index e617be8f9fb..78b2d86d10a 100644
--- a/app/controllers/projects/uploads_controller.rb
+++ b/app/controllers/projects/uploads_controller.rb
@@ -25,7 +25,8 @@ class Projects::UploadsController < Projects::ApplicationController
     return render_404 if uploader.nil? || !uploader.file.exists?
 
     disposition = uploader.image_or_video? ? 'inline' : 'attachment'
-    send_file uploader.file.path, disposition: disposition
+    data = open(uploader.file.url)
+    send_data data.read, disposition: disposition
   end
 
   private
diff --git a/app/controllers/uploads_controller.rb b/app/controllers/uploads_controller.rb
index 509f4f412ca..9fe69bd80b7 100644
--- a/app/controllers/uploads_controller.rb
+++ b/app/controllers/uploads_controller.rb
@@ -14,7 +14,8 @@ class UploadsController < ApplicationController
     end
 
     disposition = uploader.image? ? 'inline' : 'attachment'
-    send_file uploader.file.path, disposition: disposition
+    data = open(uploader.file.url)
+    send_data data.read, disposition: disposition
   end
 
   private
diff --git a/app/models/group.rb b/app/models/group.rb
index 4248e1162d8..6e573e55b44 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -103,7 +103,8 @@ class Group < Namespace
 
   def avatar_url(size = nil)
     if self[:avatar].present?
-      [gitlab_config.url, avatar.url].join
+      # [gitlab_config.url, avatar.url].join
+      avatar.url
     end
   end
 
diff --git a/app/models/project.rb b/app/models/project.rb
index bd9fcb2f3b7..9852ccdf76c 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -817,7 +817,8 @@ class Project < ActiveRecord::Base
 
   def avatar_url
     if self[:avatar].present?
-      [gitlab_config.url, avatar.url].join
+      # [gitlab_config.url, avatar.url].join
+      avatar.url
     elsif avatar_in_git
       Gitlab::Routing.url_helpers.namespace_project_avatar_url(namespace, self)
     end
diff --git a/app/models/user.rb b/app/models/user.rb
index 223d84ba916..48c1ef8ff17 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -738,7 +738,8 @@ class User < ActiveRecord::Base
 
   def avatar_url(size = nil, scale = 2)
     if self[:avatar].present?
-      [gitlab_config.url, avatar.url].join
+      # [gitlab_config.url, avatar.url].join
+      avatar.url
     else
       GravatarService.new.execute(email, size, scale)
     end
diff --git a/app/uploaders/attachment_uploader.rb b/app/uploaders/attachment_uploader.rb
index fb3b5dfecd0..f727fa8b2f7 100644
--- a/app/uploaders/attachment_uploader.rb
+++ b/app/uploaders/attachment_uploader.rb
@@ -1,7 +1,7 @@
 class AttachmentUploader < CarrierWave::Uploader::Base
   include UploaderHelper
 
-  storage :file
+  storage :aws
 
   def store_dir
     "uploads/#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}"
diff --git a/app/uploaders/avatar_uploader.rb b/app/uploaders/avatar_uploader.rb
index 71ff14a3f20..850fb6f8bff 100644
--- a/app/uploaders/avatar_uploader.rb
+++ b/app/uploaders/avatar_uploader.rb
@@ -1,7 +1,7 @@
 class AvatarUploader < CarrierWave::Uploader::Base
   include UploaderHelper
 
-  storage :file
+  storage :aws
 
   after :store, :reset_events_cache
 
diff --git a/app/uploaders/file_uploader.rb b/app/uploaders/file_uploader.rb
index 3ac6030c21c..eb0f8f3c47b 100644
--- a/app/uploaders/file_uploader.rb
+++ b/app/uploaders/file_uploader.rb
@@ -2,7 +2,7 @@ class FileUploader < CarrierWave::Uploader::Base
   include UploaderHelper
   MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)}
 
-  storage :file
+  storage :aws
 
   attr_accessor :project, :secret
 
@@ -32,7 +32,7 @@ class FileUploader < CarrierWave::Uploader::Base
   end
 
   def to_h
-    filename = image_or_video? ? self.file.basename : self.file.filename
+    filename = image_or_video? ? basename(self.file.filename) : self.file.filename
     escaped_filename = filename.gsub("]", "\\]")
 
     markdown = "[#{escaped_filename}](#{self.secure_url})"
@@ -45,6 +45,11 @@ class FileUploader < CarrierWave::Uploader::Base
     }
   end
 
+  def basename(filename)
+    elements = filename.split('.')
+    elements.first if elements.size > 1
+  end
+
   def self.generate_secret
     SecureRandom.hex
   end
diff --git a/app/uploaders/lfs_object_uploader.rb b/app/uploaders/lfs_object_uploader.rb
index 4f356dd663e..bf808d53d55 100644
--- a/app/uploaders/lfs_object_uploader.rb
+++ b/app/uploaders/lfs_object_uploader.rb
@@ -1,12 +1,16 @@
 class LfsObjectUploader < CarrierWave::Uploader::Base
-  storage :file
+  storage :aws
+
+  aws_bucket :shared
 
   def store_dir
-    "#{Gitlab.config.lfs.storage_path}/#{model.oid[0, 2]}/#{model.oid[2, 2]}"
+    #"#{Gitlab.config.lfs.storage_path}/#{model.oid[0, 2]}/#{model.oid[2, 2]}"
+    "lfs-objects/#{model.oid[0, 2]}/#{model.oid[2, 2]}"
   end
 
   def cache_dir
-    "#{Gitlab.config.lfs.storage_path}/tmp/cache"
+    # "#{Gitlab.config.lfs.storage_path}/tmp/cache"
+    'lf-objects/tmp/cache'
   end
 
   def move_to_cache
diff --git a/config/initializers/carrierwave.rb b/config/initializers/carrierwave.rb
index 1933afcbfb1..a686ce3e417 100644
--- a/config/initializers/carrierwave.rb
+++ b/config/initializers/carrierwave.rb
@@ -6,25 +6,27 @@ if File.exist?(aws_file)
   AWS_CONFIG = YAML.load(File.read(aws_file))[Rails.env]
 
   CarrierWave.configure do |config|
-    config.fog_credentials = {
-      provider: 'AWS',                                        # required
-      aws_access_key_id: AWS_CONFIG['access_key_id'],         # required
-      aws_secret_access_key: AWS_CONFIG['secret_access_key'], # required
+    config.aws_credentials = {
+      access_key_id: AWS_CONFIG['access_key_id'],         # required
+      secret_access_key: AWS_CONFIG['secret_access_key'], # required
       region: AWS_CONFIG['region'],                           # optional, defaults to 'us-east-1'
+      #host: AWS_CONFIG['host'],
+      force_path_style: true,
+      endpoint: AWS_CONFIG['endpoint']
     }
 
     # required
-    config.fog_directory = AWS_CONFIG['bucket']
+    config.aws_bucket = AWS_CONFIG['bucket']
 
     # optional, defaults to true
-    config.fog_public = false
+    config.aws_acl = 'private'
 
     # optional, defaults to {}
-    config.fog_attributes = { 'Cache-Control' => 'max-age=315576000' }
+    config.aws_attributes = { cache_control: 'max-age=315576000' }
 
     # optional time (in seconds) that authenticated urls will be valid.
     # when fog_public is false and provider is AWS or Google, defaults to 600
-    config.fog_authenticated_url_expiration = 1 << 29
+    config.aws_authenticated_url_expiration = 60 * 60 * 24 * 7
   end
 
   # Mocking Fog requests, based on: https://github.com/carrierwaveuploader/carrierwave/wiki/How-to%3A-Test-Fog-based-uploaders