Implement Build Artifacts

- Offloads uploading to GitLab Workhorse
- Use /authorize request for fast uploading
- Added backup recipes for artifacts
- Support download acceleration using X-Sendfile

10 files changed, 112 insertions, 0 deletions

diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb
index 3d9c59050ff..a9bcfc7456a 100644
--- a/app/controllers/admin/application_settings_controller.rb
+++ b/app/controllers/admin/application_settings_controller.rb
@@ -58,6 +58,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       :admin_notification_email,
       :user_oauth_applications,
       :shared_runners_enabled,
+      :max_artifacts_size,
       restricted_visibility_levels: [],
       import_sources: []
     )
diff --git a/app/controllers/projects/builds_controller.rb b/app/controllers/projects/builds_controller.rb
index 953f30e7c03..4638f77b887 100644
--- a/app/controllers/projects/builds_controller.rb
+++ b/app/controllers/projects/builds_controller.rb
@@ -3,6 +3,7 @@ class Projects::BuildsController < Projects::ApplicationController
   before_action :build, except: [:index, :cancel_all]
 
   before_action :authorize_manage_builds!, except: [:index, :show, :status]
+  before_action :authorize_download_build_artifacts!, only: [:download]
 
   layout "project"
 
@@ -51,6 +52,18 @@ class Projects::BuildsController < Projects::ApplicationController
     redirect_to build_path(build)
   end
 
+  def download
+    unless artifacts_file.file_storage?
+      return redirect_to artifacts_file.url
+    end
+
+    unless artifacts_file.exists?
+      return not_found!
+    end
+
+    send_file artifacts_file.path, disposition: 'attachment'
+  end
+
   def status
     render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha)
   end
@@ -67,6 +80,10 @@ class Projects::BuildsController < Projects::ApplicationController
     @build ||= ci_project.builds.unscoped.find_by!(id: params[:id])
   end
 
+  def artifacts_file
+    build.artifacts_file
+  end
+
   def build_path(build)
     namespace_project_build_path(build.gl_project.namespace, build.gl_project, build)
   end
@@ -76,4 +93,14 @@ class Projects::BuildsController < Projects::ApplicationController
       return page_404
     end
   end
+
+  def authorize_download_build_artifacts!
+    unless can?(current_user, :download_build_artifacts, @project)
+      if current_user.nil?
+        return authenticate_user!
+      else
+        return render_404
+      end
+    end
+  end
 end
diff --git a/app/models/ability.rb b/app/models/ability.rb
index b72178fa126..5ae28d5133e 100644
--- a/app/models/ability.rb
+++ b/app/models/ability.rb
@@ -154,6 +154,7 @@ class Ability
         :create_merge_request,
         :create_wiki,
         :manage_builds,
+        :download_build_artifacts,
         :push_code
       ]
     end
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 266045f7afa..fa7cf2464ad 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -89,6 +89,7 @@ class ApplicationSetting < ActiveRecord::Base
       restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
       import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
       shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
+      max_artifacts_size: Settings.gitlab_ci['max_artifacts_size'],
     )
   end
 
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 7f185ae7cc3..0ec7e210321 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -39,6 +39,8 @@ module Ci
     scope :ignore_failures, ->() { where(allow_failure: false) }
     scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) }
 
+    mount_uploader :artifacts_file, ArtifactUploader
+
     acts_as_taggable
 
     # To prevent db load megabytes of data from trace
@@ -217,6 +219,14 @@ module Ci
       "#{dir_to_trace}/#{id}.log"
     end
 
+    def token
+      project.token
+    end
+
+    def valid_token? token
+      project.valid_token? token
+    end
+
     def target_url
       Gitlab::Application.routes.url_helpers.
         namespace_project_build_url(gl_project.namespace, gl_project, self)
@@ -248,6 +258,13 @@ module Ci
       pending? && !any_runners_online?
     end
 
+    def download_url
+      if artifacts_file.exists?
+        Gitlab::Application.routes.url_helpers.
+          download_namespace_project_build_path(gl_project.namespace, gl_project, self)
+      end
+    end
+
     private
 
     def yaml_variables
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 7d54d83974a..d346c5d35d2 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -92,4 +92,8 @@ class CommitStatus < ActiveRecord::Base
   def show_warning?
     false
   end
+
+  def download_url
+    nil
+  end
 end
diff --git a/app/uploaders/artifact_uploader.rb b/app/uploaders/artifact_uploader.rb
new file mode 100644
index 00000000000..848e0bcbde1
--- /dev/null
+++ b/app/uploaders/artifact_uploader.rb
@@ -0,0 +1,50 @@
+# encoding: utf-8
+class ArtifactUploader < CarrierWave::Uploader::Base
+  storage :file
+
+  attr_accessor :build, :field
+
+  def self.artifacts_path
+    File.expand_path('shared/artifacts/', Rails.root)
+  end
+
+  def self.artifacts_upload_path
+    File.expand_path('shared/tmp/artifacts-uploads/', Rails.root)
+  end
+
+  def self.artifacts_cache_path
+    File.expand_path('shared/tmp/artifacts-cache/', Rails.root)
+  end
+
+  def initialize(build, field)
+    @build, @field = build, field
+  end
+
+  def artifacts_path
+    File.join(build.created_at.utc.strftime('%Y_%m'), build.project.id.to_s, build.id.to_s)
+  end
+
+  def store_dir
+    File.join(ArtifactUploader.artifacts_path, artifacts_path)
+  end
+
+  def cache_dir
+    File.join(ArtifactUploader.artifacts_cache_path, artifacts_path)
+  end
+
+  def file_storage?
+    self.class.storage == CarrierWave::Storage::File
+  end
+
+  def exists?
+    file.try(:exists?)
+  end
+
+  def move_to_cache
+    true
+  end
+
+  def move_to_store
+    true
+  end
+end
diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml
index 7253218c2e9..ddaf0e0e8ff 100644
--- a/app/views/admin/application_settings/_form.html.haml
+++ b/app/views/admin/application_settings/_form.html.haml
@@ -139,5 +139,10 @@
             = f.check_box :shared_runners_enabled
             Enable shared runners for a new projects
 
+    .form-group
+      = f.label :max_artifacts_size, 'Maximum artifacts size (MB)', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :max_artifacts_size, class: 'form-control'
+
   .form-actions
     = f.submit 'Save', class: 'btn btn-primary'
diff --git a/app/views/projects/builds/show.html.haml b/app/views/projects/builds/show.html.haml
index 3374d5432a5..7661452e6ec 100644
--- a/app/views/projects/builds/show.html.haml
+++ b/app/views/projects/builds/show.html.haml
@@ -87,6 +87,9 @@
             Test coverage
           %h1 #{@build.coverage}%
 
+      - if current_user && can?(current_user, :download_build_artifacts, @project) && @build.download_url
+        .build-widget.center
+          = link_to "Download artifacts", @build.download_url, class: 'btn btn-sm btn-primary'
 
       .build-widget
         %h4.title
diff --git a/app/views/projects/commit_statuses/_commit_status.html.haml b/app/views/projects/commit_statuses/_commit_status.html.haml
index c255559b88c..9a0e7bff3f1 100644
--- a/app/views/projects/commit_statuses/_commit_status.html.haml
+++ b/app/views/projects/commit_statuses/_commit_status.html.haml
@@ -61,6 +61,9 @@
 
   %td
     .pull-right
+      - if current_user && can?(current_user, :download_build_artifacts, @project) && commit_status.download_url
+        = link_to commit_status.download_url, title: 'Download artifacts' do
+          %i.fa.fa-download
       - if current_user && can?(current_user, :manage_builds, commit_status.gl_project)
         - if commit_status.active?
           - if commit_status.cancel_url