Merge branch 'master' into 'master'

update fog to 1.25.0 and add multipart upload support

This merge will fix backup uploading problems to s3, closing gitlab-org/gitlab-ce#1059

See merge request !789

5 files changed, 74 insertions, 17 deletions

diff --git a/Gemfile b/Gemfile
index 0d91de7e667..37871859517 100644
--- a/Gemfile
+++ b/Gemfile
@@ -81,7 +81,7 @@ gem "carrierwave"
 gem 'dropzonejs-rails'
 
 # for aws storage
-gem "fog", "~> 1.14"
+gem "fog", "~> 1.25.0"
 gem "unf"
 
 # Authorization
diff --git a/Gemfile.lock b/Gemfile.lock
index 441fe69e8a4..7590b7f9c52 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,7 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    CFPropertyList (2.3.1)
     RedCloth (4.2.9)
     ace-rails-ap (2.0.1)
     actionmailer (4.1.9)
@@ -163,7 +164,7 @@ GEM
     erubis (2.7.0)
     escape_utils (0.2.4)
     eventmachine (1.0.4)
-    excon (0.32.1)
+    excon (0.45.3)
     execjs (2.5.2)
     expression_parser (0.9.0)
     factory_girl (4.3.0)
@@ -178,29 +179,69 @@ GEM
     fastercsv (1.5.5)
     ffaker (2.0.0)
     ffi (1.9.8)
-    fog (1.21.0)
-      fog-brightbox
-      fog-core (~> 1.21, >= 1.21.1)
+    fission (0.5.0)
+      CFPropertyList (~> 2.2)
+    fog (1.25.0)
+      fog-brightbox (~> 0.4)
+      fog-core (~> 1.25)
       fog-json
+      fog-profitbricks
+      fog-radosgw (>= 0.0.2)
+      fog-sakuracloud (>= 0.0.4)
+      fog-softlayer
+      fog-terremark
+      fog-vmfusion
+      fog-voxel
+      fog-xml (~> 0.1.1)
+      ipaddress (~> 0.5)
       nokogiri (~> 1.5, >= 1.5.11)
-    fog-brightbox (0.0.1)
-      fog-core
+      opennebula
+    fog-brightbox (0.7.1)
+      fog-core (~> 1.22)
       fog-json
-    fog-core (1.21.1)
+      inflecto (~> 0.0.2)
+    fog-core (1.30.0)
       builder
-      excon (~> 0.32)
-      formatador (~> 0.2.0)
+      excon (~> 0.45)
+      formatador (~> 0.2)
       mime-types
       net-scp (~> 1.1)
       net-ssh (>= 2.1.3)
-    fog-json (1.0.0)
-      multi_json (~> 1.0)
+    fog-json (1.0.2)
+      fog-core (~> 1.0)
+      multi_json (~> 1.10)
+    fog-profitbricks (0.0.3)
+      fog-core
+      fog-xml
+      nokogiri
+    fog-radosgw (0.0.4)
+      fog-core (>= 1.21.0)
+      fog-json
+      fog-xml (>= 0.0.1)
+    fog-sakuracloud (1.0.1)
+      fog-core
+      fog-json
+    fog-softlayer (0.4.6)
+      fog-core
+      fog-json
+    fog-terremark (0.1.0)
+      fog-core
+      fog-xml
+    fog-vmfusion (0.1.0)
+      fission
+      fog-core
+    fog-voxel (0.1.0)
+      fog-core
+      fog-xml
+    fog-xml (0.1.2)
+      fog-core
+      nokogiri (~> 1.5, >= 1.5.11)
     font-awesome-rails (4.2.0.0)
       railties (>= 3.2, < 5.0)
     foreman (0.63.0)
       dotenv (>= 0.7)
       thor (>= 0.13.6)
-    formatador (0.2.4)
+    formatador (0.2.5)
     gemnasium-gitlab-service (0.2.6)
       rugged (~> 0.21)
     gemojione (2.0.0)
@@ -287,6 +328,8 @@ GEM
     i18n (0.7.0)
     ice_cube (0.11.1)
     ice_nine (0.10.0)
+    inflecto (0.0.2)
+    ipaddress (0.8.0)
     jquery-atwho-rails (1.0.1)
     jquery-rails (3.1.2)
       railties (>= 3.0, < 5.0)
@@ -328,9 +371,9 @@ GEM
     multipart-post (1.2.0)
     mysql2 (0.3.16)
     net-ldap (0.11)
-    net-scp (1.1.2)
+    net-scp (1.2.1)
       net-ssh (>= 2.6.5)
-    net-ssh (2.8.0)
+    net-ssh (2.9.2)
     newrelic_rpm (3.9.4.245)
     nokogiri (1.6.6.2)
       mini_portile (~> 0.6.0)
@@ -379,6 +422,10 @@ GEM
     omniauth-twitter (1.0.1)
       multi_json (~> 1.3)
       omniauth-oauth (~> 1.0)
+    opennebula (4.12.1)
+      json
+      nokogiri
+      rbvmomi
     org-ruby (0.9.12)
       rubypants (~> 0.2)
     orm_adapter (0.5.0)
@@ -446,6 +493,10 @@ GEM
     rb-fsevent (0.9.4)
     rb-inotify (0.9.5)
       ffi (>= 0.5.0)
+    rbvmomi (1.8.2)
+      builder
+      nokogiri (>= 1.4.1)
+      trollop
     rdoc (3.12.2)
       json (~> 1.4)
     redcarpet (3.2.3)
@@ -621,6 +672,7 @@ GEM
       multi_json (~> 1.7)
       twitter-stream (~> 0.1)
     tins (0.13.1)
+    trollop (2.1.2)
     turbolinks (2.5.3)
       coffee-rails
     twitter-stream (0.1.16)
@@ -704,7 +756,7 @@ DEPENDENCIES
   enumerize
   factory_girl_rails
   ffaker (~> 2.0.0)
-  fog (~> 1.14)
+  fog (~> 1.25.0)
   font-awesome-rails (~> 4.2)
   foreman
   gemnasium-gitlab-service (~> 0.2)
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 787b3ccfc56..f48c99fd901 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -252,6 +252,9 @@ production: &base
     #     aws_secret_access_key: 'secret123'
     #   # The remote 'directory' to store your backups. For S3, this would be the bucket name.
     #   remote_directory: 'my.s3.bucket'
+    #   # Use multipart uploads when file size reaches 100MB, see
+    #   #  http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html
+    #   multipart_chunk_size: 104857600
 
   ## GitLab Shell settings
   gitlab_shell:
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 1bd14a3a89f..c2c3c5bfde7 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -174,6 +174,7 @@ Settings.backup['upload'] ||= Settingslogic.new({ 'remote_directory' => nil, 'co
 if Settings.backup['upload']['connection']
   Settings.backup['upload']['connection'] = Hash[Settings.backup['upload']['connection'].map { |k, v| [k.to_sym, v] }]
 end
+Settings.backup['upload']['multipart_chunk_size'] ||= 104857600
 
 #
 # Git
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index b69aebf9fe1..6fa2079d1a8 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -46,7 +46,8 @@ module Backup
       connection = ::Fog::Storage.new(connection_settings)
       directory = connection.directories.get(remote_directory)
 
-      if directory.files.create(key: tar_file, body: File.open(tar_file), public: false)
+      if directory.files.create(key: tar_file, body: File.open(tar_file), public: false,
+          multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size)
         $progress.puts "done".green
       else
         puts "uploading backup to #{remote_directory} failed".red