diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/api/helpers.rb | 44 | ||||
| -rw-r--r-- | lib/backup/artifacts.rb | 13 | ||||
| -rw-r--r-- | lib/backup/manager.rb | 2 | ||||
| -rw-r--r-- | lib/ci/api/api.rb | 1 | ||||
| -rw-r--r-- | lib/ci/api/builds.rb | 102 | ||||
| -rw-r--r-- | lib/ci/api/entities.rb | 7 | ||||
| -rw-r--r-- | lib/ci/api/helpers.rb | 11 | ||||
| -rw-r--r-- | lib/ci/gitlab_ci_yaml_processor.rb | 9 | ||||
| -rw-r--r-- | lib/file_streamer.rb | 16 | ||||
| -rw-r--r-- | lib/gitlab/current_settings.rb | 1 | ||||
| -rw-r--r-- | lib/support/nginx/gitlab | 16 | ||||
| -rw-r--r-- | lib/support/nginx/gitlab-ssl | 16 | ||||
| -rw-r--r-- | lib/tasks/gitlab/backup.rake | 21 | ||||
| -rw-r--r-- | lib/uploaded_file.rb | 37 |
14 files changed, 293 insertions, 3 deletions
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 652bdf9b278..b980cd8391e 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -133,6 +133,12 @@ module API authorize! :admin_project, user_project end + def require_gitlab_workhorse! + unless headers['Gitlab-Git-Http-Server'].present? || headers['GitLab-Git-HTTP-Server'].present? + forbidden!('Request should be executed via GitLab Workhorse') + end + end + def can?(object, action, subject) abilities.allowed?(object, action, subject) end @@ -234,6 +240,10 @@ module API render_api_error!(message || '409 Conflict', 409) end + def file_to_large! + render_api_error!('413 Request Entity Too Large', 413) + end + def render_validation_error!(model) if model.errors.any? render_api_error!(model.errors.messages || '400 Bad Request', 400) @@ -282,6 +292,40 @@ module API end end + # file helpers + + def uploaded_file!(uploads_path) + required_attributes! [:file] + + # sanitize file paths + # this requires for all paths to exist + uploads_path = File.realpath(uploads_path) + file_path = File.realpath(params[:file]) + bad_request!('Bad file path') unless file_path.start_with?(uploads_path) + + UploadedFile.new( + file_path, + params[:filename], + params[:filetype] || 'application/octet-stream', + ) + end + + def present_file!(path, filename, content_type = 'application/octet-stream') + filename ||= File.basename(path) + header['Content-Disposition'] = "attachment; filename=#{filename}" + header['Content-Transfer-Encoding'] = 'binary' + content_type content_type + + # Support download acceleration + case headers['X-Sendfile-Type'] + when 'X-Sendfile' + header['X-Sendfile'] = path + body + else + file FileStreamer.new(path) + end + end + private def add_pagination_headers(paginated, per_page) diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb new file mode 100644 index 00000000000..51fa3867e67 --- /dev/null +++ b/lib/backup/artifacts.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Artifacts < Files + def initialize + super('artifacts', ArtifactUploader.artifacts_path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index f011fd03de0..9e15d5411a1 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -150,7 +150,7 @@ module Backup private def backup_contents - folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] + folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"] end def folders_to_backup diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 0a4cbf69b63..07e68216d7f 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -27,6 +27,7 @@ module Ci helpers Helpers helpers ::API::Helpers + helpers Gitlab::CurrentSettings mount Builds mount Commits diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 83ca1e6481c..622849c4b11 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -47,6 +47,108 @@ module Ci build.drop end end + + # Authorize artifacts uploading for build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # size (optional) - the size of uploaded file + # Example Request: + # POST /builds/:id/artifacts/authorize + post ":id/artifacts/authorize" do + require_gitlab_workhorse! + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + forbidden!('build is not running') unless build.running? + + if params[:filesize] + file_size = params[:filesize].to_i + file_to_large! unless file_size < max_artifacts_size + end + + status 200 + { temp_path: ArtifactUploader.artifacts_upload_path } + end + + # Upload artifacts to build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Headers: + # Content-Type - File content type + # Content-Disposition - File media type and real name + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Body: + # The file content + # + # Parameters (set by GitLab Workhorse): + # file - path to locally stored body (generated by Workhorse) + # filename - real filename as send in Content-Disposition + # filetype - real content type as send in Content-Type + # filesize - real file size as send in Content-Length + # Example Request: + # POST /builds/:id/artifacts + post ":id/artifacts" do + require_gitlab_workhorse! + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + forbidden!('build is not running') unless build.running? + + file = uploaded_file!(ArtifactUploader.artifacts_upload_path) + file_to_large! unless file.size < max_artifacts_size + + if build.update_attributes(artifacts_file: file) + present build, with: Entities::Build + else + render_validation_error!(build) + end + end + + # Download the artifacts file from build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Headers: + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Example Request: + # GET /builds/:id/artifacts + get ":id/artifacts" do + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + artifacts_file = build.artifacts_file + + unless artifacts_file.file_storage? + return redirect_to build.artifacts_file.url + end + + unless artifacts_file.exists? + not_found! + end + + present_file!(artifacts_file.path, artifacts_file.filename) + end + + # Remove the artifacts file from build + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Headers: + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Example Request: + # DELETE /builds/:id/artifacts + delete ":id/artifacts" do + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + build.remove_artifacts_file! + end end end end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index b80c0b8b273..750f421872d 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -11,10 +11,16 @@ module Ci expose :builds end + class ArtifactFile < Grape::Entity + expose :filename, :size + end + class Build < Grape::Entity expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url, :before_sha, :allow_git_fetch, :project_name + expose :name, :token, :stage + expose :options do |model| model.options end @@ -24,6 +30,7 @@ module Ci end expose :variables + expose :artifacts_file, using: ArtifactFile end class Runner < Grape::Entity diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 7e4986b6af3..02502333756 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -1,6 +1,8 @@ module Ci module API module Helpers + BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN" + BUILD_TOKEN_PARAM = :token UPDATE_RUNNER_EVERY = 60 def authenticate_runners! @@ -15,6 +17,11 @@ module Ci forbidden! unless project.valid_token?(params[:project_token]) end + def authenticate_build_token!(build) + token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s + forbidden! unless token && build.valid_token?(token) + end + def update_runner_last_contact # Use a random threshold to prevent beating DB updates contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) @@ -32,6 +39,10 @@ module Ci info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) current_runner.update(info) end + + def max_artifacts_size + current_application_settings.max_artifacts_size.megabytes.to_i + end end end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 0f57a4f53ab..6f9af5388ca 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -5,7 +5,7 @@ module Ci DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables] - ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when] + ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts] attr_reader :before_script, :image, :services, :variables, :path @@ -77,7 +77,8 @@ module Ci when: job[:when] || 'on_success', options: { image: job[:image] || @image, - services: job[:services] || @services + services: job[:services] || @services, + artifacts: job[:artifacts] }.compact } end @@ -159,6 +160,10 @@ module Ci raise ValidationError, "#{name} job: except parameter should be an array of strings" end + if job[:artifacts] && !validate_array_of_strings(job[:artifacts]) + raise ValidationError, "#{name}: artifacts parameter should be an array of strings" + end + if job[:allow_failure] && !job[:allow_failure].in?([true, false]) raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" end diff --git a/lib/file_streamer.rb b/lib/file_streamer.rb new file mode 100644 index 00000000000..4e3c6d3c773 --- /dev/null +++ b/lib/file_streamer.rb @@ -0,0 +1,16 @@ +class FileStreamer #:nodoc: + attr_reader :to_path + + def initialize(path) + @to_path = path + end + + # Stream the file's contents if Rack::Sendfile isn't present. + def each + File.open(to_path, 'rb') do |file| + while chunk = file.read(16384) + yield chunk + end + end + end +end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index cd84afa31d5..2d3e32d9539 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -25,6 +25,7 @@ module Gitlab session_expire_delay: Settings.gitlab['session_expire_delay'], import_sources: Settings.gitlab['import_sources'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], + max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'], ) end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index e767027dc29..e511d5e4b4b 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -131,6 +131,22 @@ server { return 418; } + # Build artifacts should be submitted to this location + location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + # Build artifacts should be submitted to this location + location ~ /ci/api/v1/builds/[0-9]+/artifacts { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + location @gitlab-workhorse { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 4d31e31f8d5..47b1ec8cb0c 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -178,6 +178,22 @@ server { return 418; } + # Build artifacts should be submitted to this location + location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + # Build artifacts should be submitted to this location + location ~ /ci/api/v1/builds/[0-9]+/artifacts { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + location @gitlab-workhorse { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index f20c7f71ba5..3c46bcea40e 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -12,6 +12,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:repo:create"].invoke Rake::Task["gitlab:backup:uploads:create"].invoke Rake::Task["gitlab:backup:builds:create"].invoke + Rake::Task["gitlab:backup:artifacts:create"].invoke backup = Backup::Manager.new backup.pack @@ -32,6 +33,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories") Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") + Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts") Rake::Task["gitlab:shell:setup"].invoke backup.cleanup @@ -113,6 +115,25 @@ namespace :gitlab do end end + namespace :artifacts do + task create: :environment do + $progress.puts "Dumping artifacts ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") + $progress.puts "[SKIPPED]".cyan + else + Backup::Artifacts.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring artifacts ... ".blue + Backup::Artifacts.new.restore + $progress.puts "done".green + end + end + def configure_cron_mode if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb new file mode 100644 index 00000000000..d4291f012d3 --- /dev/null +++ b/lib/uploaded_file.rb @@ -0,0 +1,37 @@ +require "tempfile" +require "fileutils" + +# Taken from: Rack::Test::UploadedFile +class UploadedFile + + # The filename, *not* including the path, of the "uploaded" file + attr_reader :original_filename + + # The tempfile + attr_reader :tempfile + + # The content type of the "uploaded" file + attr_accessor :content_type + + def initialize(path, filename, content_type = "text/plain") + raise "#{path} file does not exist" unless ::File.exist?(path) + + @content_type = content_type + @original_filename = filename || ::File.basename(path) + @tempfile = File.new(path, 'rb') + end + + def path + @tempfile.path + end + + alias_method :local_path, :path + + def method_missing(method_name, *args, &block) #:nodoc: + @tempfile.__send__(method_name, *args, &block) + end + + def respond_to?(method_name, include_private = false) #:nodoc: + @tempfile.respond_to?(method_name, include_private) || super + end +end |