diff options
author | Kamil Trzcinski <ayufan@ayufan.eu> | 2017-05-04 23:02:51 +0200 |
---|---|---|
committer | Kamil Trzcinski <ayufan@ayufan.eu> | 2017-06-01 16:49:55 +0200 |
commit | 82cc3c63294fd6eddd2675d6e384b456556dcc6e (patch) | |
tree | d0bd6cf2b3567cdf994629fc823bf315633adfe4 | |
parent | 6185d12c183b539ea06ab3550b2c21045d169ca4 (diff) | |
download | gitlab-ce-store-artifacts-on-object-storage-by-default.tar.gz |
Fix migrate_artifacts.rake task (+1 squashed commit)store-artifacts-on-object-storage-by-default
Squashed commits:
[cd77fee] Migrate artifacts to a new storage
- Store information in database what storage is being used by artifacts,
- Artifacts are now stored as `artifacts/project_id/commit_id/job_id`.
-rw-r--r-- | app/models/ci/build.rb | 25 | ||||
-rw-r--r-- | app/services/projects/update_pages_service.rb | 10 | ||||
-rw-r--r-- | app/uploaders/artifact_uploader.rb | 18 | ||||
-rw-r--r-- | app/uploaders/concerns/object_storeable.rb | 111 | ||||
-rw-r--r-- | app/uploaders/gitlab_uploader.rb | 4 | ||||
-rw-r--r-- | app/views/projects/artifacts/_tree_file.html.haml | 9 | ||||
-rw-r--r-- | app/views/projects/jobs/_sidebar.html.haml | 2 | ||||
-rw-r--r-- | config/application.rb | 3 | ||||
-rw-r--r-- | config/gitlab.yml.example | 7 | ||||
-rw-r--r-- | db/migrate/20170504203205_add_file_storage_to_ci_build.rb | 9 | ||||
-rw-r--r-- | db/schema.rb | 3 | ||||
-rw-r--r-- | doc/administration/job_artifacts.md | 11 | ||||
-rw-r--r-- | lib/api/helpers.rb | 9 | ||||
-rw-r--r-- | lib/api/runner.rb | 9 | ||||
-rw-r--r-- | lib/ci/api/builds.rb | 8 | ||||
-rw-r--r-- | lib/tasks/migrate/migrate_artifacts.rake | 17 | ||||
-rw-r--r-- | lib/uploaded_file.rb | 6 |
17 files changed, 228 insertions, 33 deletions
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 63b72b8e6a9..c2a28f50c9d 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -50,6 +50,11 @@ module Ci after_save :update_project_statistics, if: :artifacts_size_changed? after_destroy :update_project_statistics + enum artifacts_storage: { + artifacts_storage_undefined: nil, + artifacts_storage_upgraded: 1, + } + class << self # This is needed for url_for to work, # as the controller is JobsController @@ -284,17 +289,27 @@ module Ci !artifacts_expired? && artifacts_file.exists? end + def browsable_artifacts? + artifacts_metadata? + end + + def downloadable_single_artifacts_file? + artifacts_metadata? && artifacts_file.local_file? + end + def artifacts_metadata? artifacts? && artifacts_metadata.exists? end def artifacts_metadata_entry(path, **options) - metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( - artifacts_metadata.path, - path, - **options) + artifacts_metadata.use_file do |metadata_path| + metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( + metadata_path, + path, + **options) - metadata.to_entry + metadata.to_entry + end end def erase_artifacts! diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index 17cf71cf098..ee82408241d 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -96,8 +96,10 @@ module Projects # -n never overwrite existing files # We add * to end of SITE_PATH, because we want to extract SITE_PATH and all subdirectories site_path = File.join(SITE_PATH, '*') - unless system(*%W(unzip -n #{artifacts} #{site_path} -d #{temp_path})) - raise 'pages failed to extract' + build.artifacts_file.use_file do |artifacts_path| + unless system(*%W(unzip -n #{artifacts_path} #{site_path} -d #{temp_path})) + raise 'pages failed to extract' + end end end @@ -152,10 +154,6 @@ module Projects build.ref end - def artifacts - build.artifacts_file.path - end - def latest_sha project.commit(build.ref).try(:sha).to_s end diff --git a/app/uploaders/artifact_uploader.rb b/app/uploaders/artifact_uploader.rb index 3bc0408f557..c3455372384 100644 --- a/app/uploaders/artifact_uploader.rb +++ b/app/uploaders/artifact_uploader.rb @@ -1,5 +1,5 @@ class ArtifactUploader < GitlabUploader - storage :file + include ObjectStoreable attr_reader :job, :field @@ -16,11 +16,23 @@ class ArtifactUploader < GitlabUploader end def store_dir - default_local_path + if file_storage? + default_local_path + else + default_path + end end def cache_dir - File.join(self.class.local_artifacts_store, 'tmp/cache') + if file_cache_storage? + File.join(self.local_artifacts_store, 'tmp/cache') + else + 'tmp/cache' + end + end + + def migrate! + # TODO end private diff --git a/app/uploaders/concerns/object_storeable.rb b/app/uploaders/concerns/object_storeable.rb new file mode 100644 index 00000000000..e7c4044bc22 --- /dev/null +++ b/app/uploaders/concerns/object_storeable.rb @@ -0,0 +1,111 @@ +module ObjectStoreable + extend ActiveSupport::Concern + + module ClassMethods + def use_object_store? + @storage_options.object_store.enabled + end + + def storage_options(options) + @storage_options = options + + class_eval do + storage use_object_store? ? :fog : :file + end + end + end + + def fog_directory + return super unless use_object_store? + + @storage_options.bucket + end + + # Override the credentials + def fog_credentials + return super unless use_object_store? + + { + provider: @storage_options.provider, + aws_access_key_id: @storage_options.access_key_id, + aws_secret_access_key: @storage_options.secret_access_key, + region: @storage_options.region, + endpoint: @storage_options.endpoint, + path_style: true + } + end + + def fog_public + false + end + + def use_object_store? + @storage_options.object_store.enabled + end + + def move_to_store + !use_object_store? + end + + def move_to_cache + !use_object_store? + end + + def use_file + if use_object_store? + return yield path + end + + begin + cache_stored_file! + yield cache_path + ensure + cache_storage.delete_dir!(cache_path(nil)) + end + end + + def upload_authorize + self.cache_id = CarrierWave.generate_cache_id + self.original_filename = SecureRandom.hex + + result = { TempPath: cache_path } + + use_cache_object_storage do + expire_at = ::Fog::Time.now + fog_authenticated_url_expiration + result[:ObjectStore] = { + ObjectID: cache_name, + StoreURL: storage.connection.put_object_url( + fog_directory, cache_path, expire_at) + } + end + + result + end + + def cache!(new_file = nil) + unless retrive_uploaded_file!(new_file&.object_id, new_file.original_filename) + super + end + end + + def cache_storage + if @use_storage_for_cache || cached? && remote_file? + storage + else + super + end + end + + def retrive_uploaded_file!(identifier, filename) + return unless identifier + return unless filename + return unless use_object_store? + + @use_storage_for_cache = true + + retrieve_from_cache!(identifier) + @filename = filename + ensure + @use_storage_for_cache = false + end +end diff --git a/app/uploaders/gitlab_uploader.rb b/app/uploaders/gitlab_uploader.rb index 02afddb8c6a..b76f7b97e9b 100644 --- a/app/uploaders/gitlab_uploader.rb +++ b/app/uploaders/gitlab_uploader.rb @@ -38,6 +38,10 @@ class GitlabUploader < CarrierWave::Uploader::Base self.file.path.sub("#{root}/", '') end + def use_file + yield path + end + def exists? file.try(:exists?) end diff --git a/app/views/projects/artifacts/_tree_file.html.haml b/app/views/projects/artifacts/_tree_file.html.haml index ea0b43b85cf..28f7a52df25 100644 --- a/app/views/projects/artifacts/_tree_file.html.haml +++ b/app/views/projects/artifacts/_tree_file.html.haml @@ -1,10 +1,13 @@ -- path_to_file = file_namespace_project_job_artifacts_path(@project.namespace, @project, @build, path: file.path) +- path_to_file = file_namespace_project_job_artifacts_path(@project.namespace, @project, @build, path: file.path) if @build.downloadable_single_artifacts_file? %tr.tree-item{ 'data-link' => path_to_file } - blob = file.blob %td.tree-item-file-name = tree_icon('file', blob.mode, blob.name) - = link_to path_to_file do - %span.str-truncated= blob.name + %span.str-truncated + - if path_to_file + = link_to file.name, path_to_file + - else + = file.name %td = number_to_human_size(blob.size, precision: 2) diff --git a/app/views/projects/jobs/_sidebar.html.haml b/app/views/projects/jobs/_sidebar.html.haml index 3e83142377b..51ed021616f 100644 --- a/app/views/projects/jobs/_sidebar.html.haml +++ b/app/views/projects/jobs/_sidebar.html.haml @@ -36,7 +36,7 @@ = link_to download_namespace_project_job_artifacts_path(@project.namespace, @project, @build), rel: 'nofollow', download: '', class: 'btn btn-sm btn-default' do Download - - if @build.artifacts_metadata? + - if @build.browsable_artifacts? = link_to browse_namespace_project_job_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do Browse diff --git a/config/application.rb b/config/application.rb index b0533759252..5ec9d685d8e 100644 --- a/config/application.rb +++ b/config/application.rb @@ -26,7 +26,8 @@ module Gitlab #{config.root}/app/models/members #{config.root}/app/models/project_services #{config.root}/app/workers/concerns - #{config.root}/app/services/concerns)) + #{config.root}/app/services/concerns + #{config.root}/app/uploaders/concerns)) config.generators.templates.push("#{config.root}/generator_templates") diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 6c1c1f8c041..b389a637b28 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -138,6 +138,13 @@ production: &base enabled: true # The location where build artifacts are stored (default: shared/artifacts). # path: shared/artifacts + object_store: + enabled: false + provider: AWS # Only AWS supported at the moment + access_key_id: VXKLW2P7WP83RM3OQAYU + secret_access_key: hEm7WuxW3Qct9tsxNqSw+iyP26fcCacz78vErkiI + bucket: docker + region: eu-central-1 ## Git LFS lfs: diff --git a/db/migrate/20170504203205_add_file_storage_to_ci_build.rb b/db/migrate/20170504203205_add_file_storage_to_ci_build.rb new file mode 100644 index 00000000000..292ad8ed08c --- /dev/null +++ b/db/migrate/20170504203205_add_file_storage_to_ci_build.rb @@ -0,0 +1,9 @@ +class AddArtifactsFileStorageToCiBuild < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :ci_builds, :artifacts_storage, :integer + end +end diff --git a/db/schema.rb b/db/schema.rb index bac8f95ce3b..0ec8054e72d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -233,6 +233,7 @@ ActiveRecord::Schema.define(version: 20170525174156) do t.string "coverage_regex" t.integer "auto_canceled_by_id" t.boolean "retried" + t.integer "artifacts_storage" end add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree @@ -1492,4 +1493,4 @@ ActiveRecord::Schema.define(version: 20170525174156) do add_foreign_key "trending_projects", "projects", on_delete: :cascade add_foreign_key "u2f_registrations", "users" add_foreign_key "web_hook_logs", "web_hooks", on_delete: :cascade -end
\ No newline at end of file +end diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index 7b0610ae414..662f32c1d4f 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -82,6 +82,17 @@ _The artifacts are stored by default in 1. Save the file and [restart GitLab][] for the changes to take effect. +--- + +**Using Object Store** + +The previously mentioned methods use the local disk to store artifacts. However, +there is the option to use object stores like AWS' S3. To do this, set the +`object_store` flag to true in your `gitlab.rb`. This relies on valid AWS +credentials to be configured already. Please note, that enabling this feature +will have the effect that artifacts are _not_ browsable anymore through the web +interface. + ## Set the maximum file size of the artifacts Provided the artifacts are enabled, you can change the maximum file size of the diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 81f6fc3201d..49972136a78 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -273,7 +273,7 @@ module API # file helpers - def uploaded_file(field, uploads_path) + def uploaded_file(uploader, field) if params[field] bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) return params[field] @@ -284,14 +284,15 @@ module API # sanitize file paths # this requires all paths to exist required_attributes! %W(#{field}.path) - uploads_path = File.realpath(uploads_path) + artifacts_store = File.realpath(uploader.local_artifacts_store) file_path = File.realpath(params["#{field}.path"]) - bad_request!('Bad file path') unless file_path.start_with?(uploads_path) + bad_request!('Bad file path') unless file_path.start_with?(artifacts_store) UploadedFile.new( file_path, params["#{field}.name"], - params["#{field}.type"] || 'application/octet-stream' + params["#{field}.type"] || 'application/octet-stream', + params["#{field}.object_id"] ) end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 3fd0536dadd..69b1b58934f 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -181,7 +181,7 @@ module API status 200 content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE - Gitlab::Workhorse.artifact_upload_ok + job.artifacts_file.upload_authorize end desc 'Upload artifacts for job' do @@ -199,6 +199,7 @@ module API optional :file, type: File, desc: %q(Artifact's file) optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse)) + optional 'file.object_id', type: String, desc: %q(object_id as send by authorize (generated by Workhorse)) optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse)) optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse)) @@ -210,13 +211,13 @@ module API job = authenticate_job! forbidden!('Job is not running!') unless job.running? - artifacts_upload_path = ArtifactUploader.artifacts_upload_path - artifacts = uploaded_file(:file, artifacts_upload_path) - metadata = uploaded_file(:metadata, artifacts_upload_path) + artifacts = uploaded_file(job.artifacts_file, :file) + metadata = uploaded_file(job.artifacts_metadata, :metadata) bad_request!('Missing artifacts file!') unless artifacts file_to_large! unless artifacts.size < max_artifacts_size + job.artifacts_storage_upgraded! job.artifacts_file = artifacts job.artifacts_metadata = metadata job.artifacts_expire_in = params['expire_in'] || diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 2285ef241d7..9163ff3da39 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -124,7 +124,7 @@ module Ci status 200 content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE - Gitlab::Workhorse.artifact_upload_ok + build.artifacts_file.upload_authorize end # Upload artifacts to build - Runners only @@ -153,13 +153,13 @@ module Ci build = authenticate_build! forbidden!('Build is not running!') unless build.running? - artifacts_upload_path = ArtifactUploader.artifacts_upload_path - artifacts = uploaded_file(:file, artifacts_upload_path) - metadata = uploaded_file(:metadata, artifacts_upload_path) + artifacts = uploaded_file(build.artifacts_file, :file) + metadata = uploaded_file(build.artifacts_metadata, :metadata) bad_request!('Missing artifacts file!') unless artifacts file_to_large! unless artifacts.size < max_artifacts_size + build.artifacts_storage_upgraded! build.artifacts_file = artifacts build.artifacts_metadata = metadata build.artifacts_expire_in = diff --git a/lib/tasks/migrate/migrate_artifacts.rake b/lib/tasks/migrate/migrate_artifacts.rake new file mode 100644 index 00000000000..4d7ecafe20a --- /dev/null +++ b/lib/tasks/migrate/migrate_artifacts.rake @@ -0,0 +1,17 @@ +desc "GitLab | Migrate files for artifacts to comply with new storage format" +task migrate_artifacts: :environment do + puts 'Artifacts'.color(:yellow) + Ci::Build.joins(:project) + .with_artifacts + .where(artifacts_file_migrated: nil) + .find_each(batch_size: 100) do |issue| + begin + build.artifacts_file.migrate! + build.artifacts_metadata.migrate! + build.save! if build.changed? + print '.' + rescue + print 'F' + end + end +end diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index 41dee5fdc06..695e85a7f36 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -9,15 +9,19 @@ class UploadedFile # The tempfile attr_reader :tempfile + # The object_id for asynchronous uploads + attr_reader :object_id + # The content type of the "uploaded" file attr_accessor :content_type - def initialize(path, filename, content_type = "text/plain") + def initialize(path, filename, content_type = "text/plain", object_id = nil) raise "#{path} file does not exist" unless ::File.exist?(path) @content_type = content_type @original_filename = filename || ::File.basename(path) @tempfile = File.new(path, 'rb') + @object_id = object_id end def path |