summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/api/helpers.rb44
-rw-r--r--lib/backup/artifacts.rb13
-rw-r--r--lib/backup/manager.rb2
-rw-r--r--lib/ci/api/api.rb1
-rw-r--r--lib/ci/api/builds.rb102
-rw-r--r--lib/ci/api/entities.rb7
-rw-r--r--lib/ci/api/helpers.rb11
-rw-r--r--lib/ci/gitlab_ci_yaml_processor.rb9
-rw-r--r--lib/file_streamer.rb16
-rw-r--r--lib/gitlab/current_settings.rb1
-rw-r--r--lib/support/nginx/gitlab16
-rw-r--r--lib/support/nginx/gitlab-ssl16
-rw-r--r--lib/tasks/gitlab/backup.rake21
-rw-r--r--lib/uploaded_file.rb37
14 files changed, 293 insertions, 3 deletions
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 652bdf9b278..b980cd8391e 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -133,6 +133,12 @@ module API
authorize! :admin_project, user_project
end
+ def require_gitlab_workhorse!
+ unless headers['Gitlab-Git-Http-Server'].present? || headers['GitLab-Git-HTTP-Server'].present?
+ forbidden!('Request should be executed via GitLab Workhorse')
+ end
+ end
+
def can?(object, action, subject)
abilities.allowed?(object, action, subject)
end
@@ -234,6 +240,10 @@ module API
render_api_error!(message || '409 Conflict', 409)
end
+ def file_to_large!
+ render_api_error!('413 Request Entity Too Large', 413)
+ end
+
def render_validation_error!(model)
if model.errors.any?
render_api_error!(model.errors.messages || '400 Bad Request', 400)
@@ -282,6 +292,40 @@ module API
end
end
+ # file helpers
+
+ def uploaded_file!(uploads_path)
+ required_attributes! [:file]
+
+ # sanitize file paths
+ # this requires for all paths to exist
+ uploads_path = File.realpath(uploads_path)
+ file_path = File.realpath(params[:file])
+ bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
+
+ UploadedFile.new(
+ file_path,
+ params[:filename],
+ params[:filetype] || 'application/octet-stream',
+ )
+ end
+
+ def present_file!(path, filename, content_type = 'application/octet-stream')
+ filename ||= File.basename(path)
+ header['Content-Disposition'] = "attachment; filename=#{filename}"
+ header['Content-Transfer-Encoding'] = 'binary'
+ content_type content_type
+
+ # Support download acceleration
+ case headers['X-Sendfile-Type']
+ when 'X-Sendfile'
+ header['X-Sendfile'] = path
+ body
+ else
+ file FileStreamer.new(path)
+ end
+ end
+
private
def add_pagination_headers(paginated, per_page)
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
new file mode 100644
index 00000000000..51fa3867e67
--- /dev/null
+++ b/lib/backup/artifacts.rb
@@ -0,0 +1,13 @@
+require 'backup/files'
+
+module Backup
+ class Artifacts < Files
+ def initialize
+ super('artifacts', ArtifactUploader.artifacts_path)
+ end
+
+ def create_files_dir
+ Dir.mkdir(app_files_dir, 0700)
+ end
+ end
+end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index f011fd03de0..9e15d5411a1 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -150,7 +150,7 @@ module Backup
private
def backup_contents
- folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
+ folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"]
end
def folders_to_backup
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
index 0a4cbf69b63..07e68216d7f 100644
--- a/lib/ci/api/api.rb
+++ b/lib/ci/api/api.rb
@@ -27,6 +27,7 @@ module Ci
helpers Helpers
helpers ::API::Helpers
+ helpers Gitlab::CurrentSettings
mount Builds
mount Commits
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 83ca1e6481c..622849c4b11 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -47,6 +47,108 @@ module Ci
build.drop
end
end
+
+ # Authorize artifacts uploading for build - Runners only
+ #
+ # Parameters:
+ # id (required) - The ID of a build
+ # token (required) - The build authorization token
+ # size (optional) - the size of uploaded file
+ # Example Request:
+ # POST /builds/:id/artifacts/authorize
+ post ":id/artifacts/authorize" do
+ require_gitlab_workhorse!
+ build = Ci::Build.find_by_id(params[:id])
+ not_found! unless build
+ authenticate_build_token!(build)
+ forbidden!('build is not running') unless build.running?
+
+ if params[:filesize]
+ file_size = params[:filesize].to_i
+ file_to_large! unless file_size < max_artifacts_size
+ end
+
+ status 200
+ { temp_path: ArtifactUploader.artifacts_upload_path }
+ end
+
+ # Upload artifacts to build - Runners only
+ #
+ # Parameters:
+ # id (required) - The ID of a build
+ # token (required) - The build authorization token
+ # Headers:
+ # Content-Type - File content type
+ # Content-Disposition - File media type and real name
+ # BUILD-TOKEN (required) - The build authorization token, the same as token
+ # Body:
+ # The file content
+ #
+ # Parameters (set by GitLab Workhorse):
+ # file - path to locally stored body (generated by Workhorse)
+ # filename - real filename as send in Content-Disposition
+ # filetype - real content type as send in Content-Type
+ # filesize - real file size as send in Content-Length
+ # Example Request:
+ # POST /builds/:id/artifacts
+ post ":id/artifacts" do
+ require_gitlab_workhorse!
+ build = Ci::Build.find_by_id(params[:id])
+ not_found! unless build
+ authenticate_build_token!(build)
+ forbidden!('build is not running') unless build.running?
+
+ file = uploaded_file!(ArtifactUploader.artifacts_upload_path)
+ file_to_large! unless file.size < max_artifacts_size
+
+ if build.update_attributes(artifacts_file: file)
+ present build, with: Entities::Build
+ else
+ render_validation_error!(build)
+ end
+ end
+
+ # Download the artifacts file from build - Runners only
+ #
+ # Parameters:
+ # id (required) - The ID of a build
+ # token (required) - The build authorization token
+ # Headers:
+ # BUILD-TOKEN (required) - The build authorization token, the same as token
+ # Example Request:
+ # GET /builds/:id/artifacts
+ get ":id/artifacts" do
+ build = Ci::Build.find_by_id(params[:id])
+ not_found! unless build
+ authenticate_build_token!(build)
+ artifacts_file = build.artifacts_file
+
+ unless artifacts_file.file_storage?
+ return redirect_to build.artifacts_file.url
+ end
+
+ unless artifacts_file.exists?
+ not_found!
+ end
+
+ present_file!(artifacts_file.path, artifacts_file.filename)
+ end
+
+ # Remove the artifacts file from build
+ #
+ # Parameters:
+ # id (required) - The ID of a build
+ # token (required) - The build authorization token
+ # Headers:
+ # BUILD-TOKEN (required) - The build authorization token, the same as token
+ # Example Request:
+ # DELETE /builds/:id/artifacts
+ delete ":id/artifacts" do
+ build = Ci::Build.find_by_id(params[:id])
+ not_found! unless build
+ authenticate_build_token!(build)
+ build.remove_artifacts_file!
+ end
end
end
end
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
index b80c0b8b273..750f421872d 100644
--- a/lib/ci/api/entities.rb
+++ b/lib/ci/api/entities.rb
@@ -11,10 +11,16 @@ module Ci
expose :builds
end
+ class ArtifactFile < Grape::Entity
+ expose :filename, :size
+ end
+
class Build < Grape::Entity
expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url,
:before_sha, :allow_git_fetch, :project_name
+ expose :name, :token, :stage
+
expose :options do |model|
model.options
end
@@ -24,6 +30,7 @@ module Ci
end
expose :variables
+ expose :artifacts_file, using: ArtifactFile
end
class Runner < Grape::Entity
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index 7e4986b6af3..02502333756 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -1,6 +1,8 @@
module Ci
module API
module Helpers
+ BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN"
+ BUILD_TOKEN_PARAM = :token
UPDATE_RUNNER_EVERY = 60
def authenticate_runners!
@@ -15,6 +17,11 @@ module Ci
forbidden! unless project.valid_token?(params[:project_token])
end
+ def authenticate_build_token!(build)
+ token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
+ forbidden! unless token && build.valid_token?(token)
+ end
+
def update_runner_last_contact
# Use a random threshold to prevent beating DB updates
contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
@@ -32,6 +39,10 @@ module Ci
info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"])
current_runner.update(info)
end
+
+ def max_artifacts_size
+ current_application_settings.max_artifacts_size.megabytes.to_i
+ end
end
end
end
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 0f57a4f53ab..6f9af5388ca 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -5,7 +5,7 @@ module Ci
DEFAULT_STAGES = %w(build test deploy)
DEFAULT_STAGE = 'test'
ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
- ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when]
+ ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts]
attr_reader :before_script, :image, :services, :variables, :path
@@ -77,7 +77,8 @@ module Ci
when: job[:when] || 'on_success',
options: {
image: job[:image] || @image,
- services: job[:services] || @services
+ services: job[:services] || @services,
+ artifacts: job[:artifacts]
}.compact
}
end
@@ -159,6 +160,10 @@ module Ci
raise ValidationError, "#{name} job: except parameter should be an array of strings"
end
+ if job[:artifacts] && !validate_array_of_strings(job[:artifacts])
+ raise ValidationError, "#{name}: artifacts parameter should be an array of strings"
+ end
+
if job[:allow_failure] && !job[:allow_failure].in?([true, false])
raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
end
diff --git a/lib/file_streamer.rb b/lib/file_streamer.rb
new file mode 100644
index 00000000000..4e3c6d3c773
--- /dev/null
+++ b/lib/file_streamer.rb
@@ -0,0 +1,16 @@
+class FileStreamer #:nodoc:
+ attr_reader :to_path
+
+ def initialize(path)
+ @to_path = path
+ end
+
+ # Stream the file's contents if Rack::Sendfile isn't present.
+ def each
+ File.open(to_path, 'rb') do |file|
+ while chunk = file.read(16384)
+ yield chunk
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index cd84afa31d5..2d3e32d9539 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -25,6 +25,7 @@ module Gitlab
session_expire_delay: Settings.gitlab['session_expire_delay'],
import_sources: Settings.gitlab['import_sources'],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
+ max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'],
)
end
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index e767027dc29..e511d5e4b4b 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -131,6 +131,22 @@ server {
return 418;
}
+ # Build artifacts should be submitted to this location
+ location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+ client_max_body_size 0;
+ # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+ error_page 418 = @gitlab-git-http-server;
+ return 418;
+ }
+
+ # Build artifacts should be submitted to this location
+ location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+ client_max_body_size 0;
+ # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+ error_page 418 = @gitlab-git-http-server;
+ return 418;
+ }
+
location @gitlab-workhorse {
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 4d31e31f8d5..47b1ec8cb0c 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -178,6 +178,22 @@ server {
return 418;
}
+ # Build artifacts should be submitted to this location
+ location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+ client_max_body_size 0;
+ # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+ error_page 418 = @gitlab-git-http-server;
+ return 418;
+ }
+
+ # Build artifacts should be submitted to this location
+ location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+ client_max_body_size 0;
+ # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+ error_page 418 = @gitlab-git-http-server;
+ return 418;
+ }
+
location @gitlab-workhorse {
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index f20c7f71ba5..3c46bcea40e 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -12,6 +12,7 @@ namespace :gitlab do
Rake::Task["gitlab:backup:repo:create"].invoke
Rake::Task["gitlab:backup:uploads:create"].invoke
Rake::Task["gitlab:backup:builds:create"].invoke
+ Rake::Task["gitlab:backup:artifacts:create"].invoke
backup = Backup::Manager.new
backup.pack
@@ -32,6 +33,7 @@ namespace :gitlab do
Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
+ Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
Rake::Task["gitlab:shell:setup"].invoke
backup.cleanup
@@ -113,6 +115,25 @@ namespace :gitlab do
end
end
+ namespace :artifacts do
+ task create: :environment do
+ $progress.puts "Dumping artifacts ... ".blue
+
+ if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
+ $progress.puts "[SKIPPED]".cyan
+ else
+ Backup::Artifacts.new.dump
+ $progress.puts "done".green
+ end
+ end
+
+ task restore: :environment do
+ $progress.puts "Restoring artifacts ... ".blue
+ Backup::Artifacts.new.restore
+ $progress.puts "done".green
+ end
+ end
+
def configure_cron_mode
if ENV['CRON']
# We need an object we can say 'puts' and 'print' to; let's use a
diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb
new file mode 100644
index 00000000000..d4291f012d3
--- /dev/null
+++ b/lib/uploaded_file.rb
@@ -0,0 +1,37 @@
+require "tempfile"
+require "fileutils"
+
+# Taken from: Rack::Test::UploadedFile
+class UploadedFile
+
+ # The filename, *not* including the path, of the "uploaded" file
+ attr_reader :original_filename
+
+ # The tempfile
+ attr_reader :tempfile
+
+ # The content type of the "uploaded" file
+ attr_accessor :content_type
+
+ def initialize(path, filename, content_type = "text/plain")
+ raise "#{path} file does not exist" unless ::File.exist?(path)
+
+ @content_type = content_type
+ @original_filename = filename || ::File.basename(path)
+ @tempfile = File.new(path, 'rb')
+ end
+
+ def path
+ @tempfile.path
+ end
+
+ alias_method :local_path, :path
+
+ def method_missing(method_name, *args, &block) #:nodoc:
+ @tempfile.__send__(method_name, *args, &block)
+ end
+
+ def respond_to?(method_name, include_private = false) #:nodoc:
+ @tempfile.respond_to?(method_name, include_private) || super
+ end
+end