Merge branch 'artifacts' into 'master'

Implement Build Artifacts

This implements #3028 

1. It stores artifacts in shared/artifacts,
1. It adds `artifacts` to `.gitlab-ci.yml`,
1. We use GitLab Workhorse to offload artifacts uploading,
1. To download artifacts it uses GitLab Workhorse X-Sendfile extension,
1. There's one "artifact" per-build. The new upload removes previous one and creates a new one,
1. Default max artifact size is set to 100MB - this can be changed in settings.

Missing things:
1. Support for `.gitlab-ci.yml`: `artifacts: true or git-ls-files` which will upload all non tracked files,
1. Artifacts passing between builds.

GitLab Workhorse changes: https://gitlab.com/gitlab-org/gitlab-workhorse/merge_requests/5
GitLab Runner changes: https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/merge_requests/46

Syntax:
```
artifacts:
  untracked: true # default: false
  paths: # default: empty
  - bin/files
```


See merge request !1584

14 files changed, 306 insertions, 4 deletions

diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 652bdf9b278..92540ccf2b1 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -133,6 +133,12 @@ module API
       authorize! :admin_project, user_project
     end
 
+    def require_gitlab_workhorse!
+      unless env['HTTP_GITLAB_WORKHORSE'].present?
+        forbidden!('Request should be executed via GitLab Workhorse')
+      end
+    end
+
     def can?(object, action, subject)
       abilities.allowed?(object, action, subject)
     end
@@ -234,6 +240,10 @@ module API
       render_api_error!(message || '409 Conflict', 409)
     end
 
+    def file_to_large!
+      render_api_error!('413 Request Entity Too Large', 413)
+    end
+
     def render_validation_error!(model)
       if model.errors.any?
         render_api_error!(model.errors.messages || '400 Bad Request', 400)
@@ -282,6 +292,44 @@ module API
       end
     end
 
+    # file helpers
+
+    def uploaded_file!(field, uploads_path)
+      if params[field]
+        bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename)
+        return params[field]
+      end
+
+      # sanitize file paths
+      # this requires all paths to exist
+      required_attributes! %W(#{field}.path)
+      uploads_path = File.realpath(uploads_path)
+      file_path = File.realpath(params["#{field}.path"])
+      bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
+
+      UploadedFile.new(
+        file_path,
+        params["#{field}.name"],
+        params["#{field}.type"] || 'application/octet-stream',
+      )
+    end
+
+    def present_file!(path, filename, content_type = 'application/octet-stream')
+      filename ||= File.basename(path)
+      header['Content-Disposition'] = "attachment; filename=#{filename}"
+      header['Content-Transfer-Encoding'] = 'binary'
+      content_type content_type
+
+      # Support download acceleration
+      case headers['X-Sendfile-Type']
+      when 'X-Sendfile'
+        header['X-Sendfile'] = path
+        body
+      else
+        file FileStreamer.new(path)
+      end
+    end
+
     private
 
     def add_pagination_headers(paginated, per_page)
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
new file mode 100644
index 00000000000..51fa3867e67
--- /dev/null
+++ b/lib/backup/artifacts.rb
@@ -0,0 +1,13 @@
+require 'backup/files'
+
+module Backup
+  class Artifacts < Files
+    def initialize
+      super('artifacts', ArtifactUploader.artifacts_path)
+    end
+
+    def create_files_dir
+      Dir.mkdir(app_files_dir, 0700)
+    end
+  end
+end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index f011fd03de0..9e15d5411a1 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -150,7 +150,7 @@ module Backup
     private
 
     def backup_contents
-      folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
+      folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"]
     end
 
     def folders_to_backup
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
index 0a4cbf69b63..07e68216d7f 100644
--- a/lib/ci/api/api.rb
+++ b/lib/ci/api/api.rb
@@ -27,6 +27,7 @@ module Ci
 
       helpers Helpers
       helpers ::API::Helpers
+      helpers Gitlab::CurrentSettings
 
       mount Builds
       mount Commits
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 83ca1e6481c..0a586672807 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -47,6 +47,106 @@ module Ci
             build.drop
           end
         end
+
+        # Authorize artifacts uploading for build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        #   filesize (optional) - the size of uploaded file
+        # Example Request:
+        #   POST /builds/:id/artifacts/authorize
+        post ":id/artifacts/authorize" do
+          require_gitlab_workhorse!
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          forbidden!('build is not running') unless build.running?
+
+          if params[:filesize]
+            file_size = params[:filesize].to_i
+            file_to_large! unless file_size < max_artifacts_size
+          end
+
+          status 200
+          { TempPath: ArtifactUploader.artifacts_upload_path }
+        end
+
+        # Upload artifacts to build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        #   file (required) - The uploaded file
+        # Parameters (accelerated by GitLab Workhorse):
+        #   file.path - path to locally stored body (generated by Workhorse)
+        #   file.name - real filename as send in Content-Disposition
+        #   file.type - real content type as send in Content-Type
+        # Headers:
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Body:
+        #   The file content
+        #
+        # Example Request:
+        #   POST /builds/:id/artifacts
+        post ":id/artifacts" do
+          require_gitlab_workhorse!
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          forbidden!('build is not running') unless build.running?
+
+          file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path)
+          file_to_large! unless file.size < max_artifacts_size
+
+          if build.update_attributes(artifacts_file: file)
+            present build, with: Entities::Build
+          else
+            render_validation_error!(build)
+          end
+        end
+
+        # Download the artifacts file from build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        # Headers:
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Example Request:
+        #   GET /builds/:id/artifacts
+        get ":id/artifacts" do
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          artifacts_file = build.artifacts_file
+
+          unless artifacts_file.file_storage?
+            return redirect_to build.artifacts_file.url
+          end
+
+          unless artifacts_file.exists?
+            not_found!
+          end
+
+          present_file!(artifacts_file.path, artifacts_file.filename)
+        end
+
+        # Remove the artifacts file from build
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        # Headers:
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Example Request:
+        #   DELETE /builds/:id/artifacts
+        delete ":id/artifacts" do
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          build.remove_artifacts_file!
+        end
       end
     end
   end
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
index b80c0b8b273..750f421872d 100644
--- a/lib/ci/api/entities.rb
+++ b/lib/ci/api/entities.rb
@@ -11,10 +11,16 @@ module Ci
         expose :builds
       end
 
+      class ArtifactFile < Grape::Entity
+        expose :filename, :size
+      end
+
       class Build < Grape::Entity
         expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url,
           :before_sha, :allow_git_fetch, :project_name
 
+        expose :name, :token, :stage
+
         expose :options do |model|
           model.options
         end
@@ -24,6 +30,7 @@ module Ci
         end
 
         expose :variables
+        expose :artifacts_file, using: ArtifactFile
       end
 
       class Runner < Grape::Entity
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index 7e4986b6af3..02502333756 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -1,6 +1,8 @@
 module Ci
   module API
     module Helpers
+      BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN"
+      BUILD_TOKEN_PARAM = :token
       UPDATE_RUNNER_EVERY = 60
 
       def authenticate_runners!
@@ -15,6 +17,11 @@ module Ci
         forbidden! unless project.valid_token?(params[:project_token])
       end
 
+      def authenticate_build_token!(build)
+        token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
+        forbidden! unless token && build.valid_token?(token)
+      end
+
       def update_runner_last_contact
         # Use a random threshold to prevent beating DB updates
         contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
@@ -32,6 +39,10 @@ module Ci
         info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"])
         current_runner.update(info)
       end
+
+      def max_artifacts_size
+        current_application_settings.max_artifacts_size.megabytes.to_i
+      end
     end
   end
 end
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 0f57a4f53ab..2e2209031ee 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -5,7 +5,7 @@ module Ci
     DEFAULT_STAGES = %w(build test deploy)
     DEFAULT_STAGE = 'test'
     ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
-    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when]
+    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts]
 
     attr_reader :before_script, :image, :services, :variables, :path
 
@@ -77,7 +77,8 @@ module Ci
         when: job[:when] || 'on_success',
         options: {
           image: job[:image] || @image,
-          services: job[:services] || @services
+          services: job[:services] || @services,
+          artifacts: job[:artifacts]
         }.compact
       }
     end
@@ -159,7 +160,17 @@ module Ci
         raise ValidationError, "#{name} job: except parameter should be an array of strings"
       end
 
-      if job[:allow_failure] && !job[:allow_failure].in?([true, false])
+      if job[:artifacts]
+        if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
+          raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
+        end
+
+        if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
+          raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
+        end
+      end
+
+      if job[:allow_failure] && !validate_boolean(job[:allow_failure])
         raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
       end
 
@@ -182,6 +193,10 @@ module Ci
       value.is_a?(String) || value.is_a?(Symbol)
     end
 
+    def validate_boolean(value)
+      value.in?([true, false])
+    end
+
     def process?(only_params, except_params, ref, tag)
       if only_params.present?
         return false unless matching?(only_params, ref, tag)
diff --git a/lib/file_streamer.rb b/lib/file_streamer.rb
new file mode 100644
index 00000000000..4e3c6d3c773
--- /dev/null
+++ b/lib/file_streamer.rb
@@ -0,0 +1,16 @@
+class FileStreamer #:nodoc:
+  attr_reader :to_path
+
+  def initialize(path)
+    @to_path = path
+  end
+
+  # Stream the file's contents if Rack::Sendfile isn't present.
+  def each
+    File.open(to_path, 'rb') do |file|
+      while chunk = file.read(16384)
+        yield chunk
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index cd84afa31d5..2d3e32d9539 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -25,6 +25,7 @@ module Gitlab
         session_expire_delay: Settings.gitlab['session_expire_delay'],
         import_sources: Settings.gitlab['import_sources'],
         shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
+        max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'],
       )
     end
 
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index e767027dc29..0a7a4118077 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -131,6 +131,22 @@ server {
     return 418;
   }
 
+  # Build artifacts should be submitted to this location
+  location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
+      return 418;
+  }
+
+  # Build artifacts should be submitted to this location
+  location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
+      return 418;
+  }
+
   location @gitlab-workhorse {
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 4d31e31f8d5..b463d5b6aa9 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -178,6 +178,22 @@ server {
     return 418;
   }
 
+  # Build artifacts should be submitted to this location
+  location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
+      return 418;
+  }
+
+  # Build artifacts should be submitted to this location
+  location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
+      return 418;
+  }
+
   location @gitlab-workhorse {
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index f20c7f71ba5..3c46bcea40e 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -12,6 +12,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:repo:create"].invoke
       Rake::Task["gitlab:backup:uploads:create"].invoke
       Rake::Task["gitlab:backup:builds:create"].invoke
+      Rake::Task["gitlab:backup:artifacts:create"].invoke
 
       backup = Backup::Manager.new
       backup.pack
@@ -32,6 +33,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
       Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
       Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
+      Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
       Rake::Task["gitlab:shell:setup"].invoke
 
       backup.cleanup
@@ -113,6 +115,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :artifacts do
+      task create: :environment do
+        $progress.puts "Dumping artifacts ... ".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
+          $progress.puts "[SKIPPED]".cyan
+        else
+          Backup::Artifacts.new.dump
+          $progress.puts "done".green
+        end
+      end
+
+      task restore: :environment do
+        $progress.puts "Restoring artifacts ... ".blue
+        Backup::Artifacts.new.restore
+        $progress.puts "done".green
+      end
+    end
+
     def configure_cron_mode
       if ENV['CRON']
         # We need an object we can say 'puts' and 'print' to; let's use a
diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb
new file mode 100644
index 00000000000..d4291f012d3
--- /dev/null
+++ b/lib/uploaded_file.rb
@@ -0,0 +1,37 @@
+require "tempfile"
+require "fileutils"
+
+# Taken from: Rack::Test::UploadedFile
+class UploadedFile
+
+  # The filename, *not* including the path, of the "uploaded" file
+  attr_reader :original_filename
+
+  # The tempfile
+  attr_reader :tempfile
+
+  # The content type of the "uploaded" file
+  attr_accessor :content_type
+
+  def initialize(path, filename, content_type = "text/plain")
+    raise "#{path} file does not exist" unless ::File.exist?(path)
+
+    @content_type = content_type
+    @original_filename = filename || ::File.basename(path)
+    @tempfile = File.new(path, 'rb')
+  end
+
+  def path
+    @tempfile.path
+  end
+
+  alias_method :local_path, :path
+
+  def method_missing(method_name, *args, &block) #:nodoc:
+    @tempfile.__send__(method_name, *args, &block)
+  end
+
+  def respond_to?(method_name, include_private = false) #:nodoc:
+    @tempfile.respond_to?(method_name, include_private) || super
+  end
+end