summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAhmad Sherif <me@ahmadsherif.com>2019-07-22 16:56:40 +0200
committerAhmad Sherif <me@ahmadsherif.com>2019-09-10 13:43:11 +0200
commit3c2b4a1cede956d5160ccf08d0a561bf31248161 (patch)
tree9462f59d477ffe7ac1eee0fe56cf9f343b568d1f
parentf7e7ee713aa21874bf6810d01976c2b5342c0995 (diff)
downloadgitlab-ce-3c2b4a1cede956d5160ccf08d0a561bf31248161.tar.gz
Enable serving static objects from an external storagestatic-objects-external-storage
It consists of two parts: 1. Redirecting users to the configured external storage 1. Allowing the external storage to request the static object(s) on behalf of the user by means of specific tokens Part of https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/6829
-rw-r--r--app/controllers/concerns/sessionless_authentication.rb4
-rw-r--r--app/controllers/concerns/static_object_external_storage.rb24
-rw-r--r--app/controllers/profiles_controller.rb9
-rw-r--r--app/controllers/projects/repositories_controller.rb4
-rw-r--r--app/helpers/application_helper.rb19
-rw-r--r--app/helpers/application_settings_helper.rb2
-rw-r--r--app/models/application_setting.rb8
-rw-r--r--app/models/application_setting_implementation.rb4
-rw-r--r--app/models/user.rb8
-rw-r--r--app/views/admin/application_settings/_repository_static_objects.html.haml18
-rw-r--r--app/views/admin/application_settings/repository.html.haml11
-rw-r--r--app/views/profiles/personal_access_tokens/index.html.haml20
-rw-r--r--app/views/projects/buttons/_download_links.html.haml3
-rw-r--r--changelogs/unreleased/static-objects-external-storage.yml5
-rw-r--r--config/routes/profile.rb1
-rw-r--r--db/migrate/20190722104947_add_static_object_token_to_users.rb18
-rw-r--r--db/migrate/20190722132830_add_static_objects_external_storage_columns_to_application_settings.rb14
-rw-r--r--db/migrate/20190725183432_add_index_to_index_on_static_object_token.rb21
-rw-r--r--db/schema.rb4
-rw-r--r--doc/administration/index.md1
-rw-r--r--doc/administration/static_objects_external_storage.md50
-rw-r--r--lib/gitlab/auth/request_authenticator.rb4
-rw-r--r--lib/gitlab/auth/user_auth_finders.rb22
-rw-r--r--locale/gitlab.pot33
-rw-r--r--spec/controllers/concerns/static_object_external_storage_spec.rb96
-rw-r--r--spec/controllers/projects/repositories_controller_spec.rb54
-rw-r--r--spec/features/projects/branches/download_buttons_spec.rb5
-rw-r--r--spec/features/projects/files/download_buttons_spec.rb8
-rw-r--r--spec/features/projects/show/download_buttons_spec.rb2
-rw-r--r--spec/features/projects/tags/download_buttons_spec.rb5
-rw-r--r--spec/helpers/application_helper_spec.rb37
-rw-r--r--spec/lib/gitlab/auth/user_auth_finders_spec.rb54
-rw-r--r--spec/models/application_setting_spec.rb14
-rw-r--r--spec/models/user_spec.rb10
-rw-r--r--spec/support/shared_examples/features/archive_download_buttons_shared_examples.rb59
35 files changed, 646 insertions, 5 deletions
diff --git a/app/controllers/concerns/sessionless_authentication.rb b/app/controllers/concerns/sessionless_authentication.rb
index 4304b8565ce..ba06384a37a 100644
--- a/app/controllers/concerns/sessionless_authentication.rb
+++ b/app/controllers/concerns/sessionless_authentication.rb
@@ -2,10 +2,10 @@
# == SessionlessAuthentication
#
-# Controller concern to handle PAT and RSS token authentication methods
+# Controller concern to handle PAT, RSS, and static objects token authentication methods
#
module SessionlessAuthentication
- # This filter handles personal access tokens, and atom requests with rss tokens
+ # This filter handles personal access tokens, atom requests with rss tokens, and static object tokens
def authenticate_sessionless_user!(request_format)
user = Gitlab::Auth::RequestAuthenticator.new(request).find_sessionless_user(request_format)
diff --git a/app/controllers/concerns/static_object_external_storage.rb b/app/controllers/concerns/static_object_external_storage.rb
new file mode 100644
index 00000000000..dbfe0ed3adf
--- /dev/null
+++ b/app/controllers/concerns/static_object_external_storage.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module StaticObjectExternalStorage
+ extend ActiveSupport::Concern
+
+ included do
+ include ApplicationHelper
+ end
+
+ def redirect_to_external_storage
+ return if external_storage_request?
+
+ redirect_to external_storage_url_or_path(request.fullpath, project)
+ end
+
+ def external_storage_request?
+ header_token = request.headers['X-Gitlab-External-Storage-Token']
+ return false unless header_token.present?
+
+ external_storage_token = Gitlab::CurrentSettings.static_objects_external_storage_auth_token
+ ActiveSupport::SecurityUtils.secure_compare(header_token, external_storage_token) ||
+ raise(Gitlab::Access::AccessDeniedError)
+ end
+end
diff --git a/app/controllers/profiles_controller.rb b/app/controllers/profiles_controller.rb
index 1d16ddb1608..958a24b6c0e 100644
--- a/app/controllers/profiles_controller.rb
+++ b/app/controllers/profiles_controller.rb
@@ -46,6 +46,15 @@ class ProfilesController < Profiles::ApplicationController
redirect_to profile_personal_access_tokens_path
end
+ def reset_static_object_token
+ Users::UpdateService.new(current_user, user: @user).execute! do |user|
+ user.reset_static_object_token!
+ end
+
+ redirect_to profile_personal_access_tokens_path,
+ notice: s_('Profiles|Static object token was successfully reset')
+ end
+
# rubocop: disable CodeReuse/ActiveRecord
def audit_log
@events = AuditEvent.where(entity_type: "User", entity_id: current_user.id)
diff --git a/app/controllers/projects/repositories_controller.rb b/app/controllers/projects/repositories_controller.rb
index a51759641e4..d69f9e65874 100644
--- a/app/controllers/projects/repositories_controller.rb
+++ b/app/controllers/projects/repositories_controller.rb
@@ -2,6 +2,9 @@
class Projects::RepositoriesController < Projects::ApplicationController
include ExtractsPath
+ include StaticObjectExternalStorage
+
+ prepend_before_action(only: [:archive]) { authenticate_sessionless_user!(:archive) }
# Authorize
before_action :require_non_empty_project, except: :create
@@ -9,6 +12,7 @@ class Projects::RepositoriesController < Projects::ApplicationController
before_action :assign_append_sha, only: :archive
before_action :authorize_download_code!
before_action :authorize_admin_project!, only: :create
+ before_action :redirect_to_external_storage, only: :archive, if: :static_objects_external_storage_enabled?
def create
@project.create_repository
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
index ffa5719fefb..1671aa5bd04 100644
--- a/app/helpers/application_helper.rb
+++ b/app/helpers/application_helper.rb
@@ -169,6 +169,25 @@ module ApplicationHelper
Gitlab::CurrentSettings.current_application_settings.help_page_support_url.presence || promo_url + '/getting-help/'
end
+ def static_objects_external_storage_enabled?
+ Gitlab::CurrentSettings.static_objects_external_storage_enabled?
+ end
+
+ def external_storage_url_or_path(path, project = @project)
+ return path unless static_objects_external_storage_enabled?
+
+ uri = URI(Gitlab::CurrentSettings.static_objects_external_storage_url)
+ path = URI(path) # `path` could have query parameters, so we need to split query and path apart
+
+ query = Rack::Utils.parse_nested_query(path.query)
+ query['token'] = current_user.static_object_token unless project.public?
+
+ uri.path = path.path
+ uri.query = query.to_query unless query.empty?
+
+ uri.to_s
+ end
+
def page_filter_path(options = {})
without = options.delete(:without)
diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb
index 84021d0da56..be3b67c60e8 100644
--- a/app/helpers/application_settings_helper.rb
+++ b/app/helpers/application_settings_helper.rb
@@ -168,6 +168,8 @@ module ApplicationSettingsHelper
:asset_proxy_secret_key,
:asset_proxy_url,
:asset_proxy_whitelist,
+ :static_objects_external_storage_auth_token,
+ :static_objects_external_storage_url,
:authorized_keys_enabled,
:auto_devops_enabled,
:auto_devops_domain,
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index e39d655325f..3409411c3b1 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -8,6 +8,7 @@ class ApplicationSetting < ApplicationRecord
add_authentication_token_field :runners_registration_token, encrypted: -> { Feature.enabled?(:application_settings_tokens_optional_encryption, default_enabled: true) ? :optional : :required }
add_authentication_token_field :health_check_access_token
+ add_authentication_token_field :static_objects_external_storage_auth_token
belongs_to :instance_administration_project, class_name: "Project"
@@ -211,6 +212,13 @@ class ApplicationSetting < ApplicationRecord
allow_blank: false,
if: :asset_proxy_enabled?
+ validates :static_objects_external_storage_url,
+ addressable_url: true, allow_blank: true
+
+ validates :static_objects_external_storage_auth_token,
+ presence: true,
+ if: :static_objects_external_storage_url?
+
SUPPORTED_KEY_TYPES.each do |type|
validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type }
end
diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb
index f402c0e2775..8d9597aa5a4 100644
--- a/app/models/application_setting_implementation.rb
+++ b/app/models/application_setting_implementation.rb
@@ -306,6 +306,10 @@ module ApplicationSettingImplementation
archive_builds_in_seconds.seconds.ago if archive_builds_in_seconds
end
+ def static_objects_external_storage_enabled?
+ static_objects_external_storage_url.present?
+ end
+
private
def array_to_string(arr)
diff --git a/app/models/user.rb b/app/models/user.rb
index 67d730e2fa3..75532aeebb3 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -31,6 +31,7 @@ class User < ApplicationRecord
add_authentication_token_field :incoming_email_token, token_generator: -> { SecureRandom.hex.to_i(16).to_s(36) }
add_authentication_token_field :feed_token
+ add_authentication_token_field :static_object_token
default_value_for :admin, false
default_value_for(:external) { Gitlab::CurrentSettings.user_default_external }
@@ -1437,6 +1438,13 @@ class User < ApplicationRecord
ensure_feed_token!
end
+ # Each existing user needs to have a `static_object_token`.
+ # We do this on read since migrating all existing users is not a feasible
+ # solution.
+ def static_object_token
+ ensure_static_object_token!
+ end
+
def sync_attribute?(attribute)
return true if ldap_user? && attribute == :email
diff --git a/app/views/admin/application_settings/_repository_static_objects.html.haml b/app/views/admin/application_settings/_repository_static_objects.html.haml
new file mode 100644
index 00000000000..03aa48b2282
--- /dev/null
+++ b/app/views/admin/application_settings/_repository_static_objects.html.haml
@@ -0,0 +1,18 @@
+= form_for @application_setting, url: repository_admin_application_settings_path(anchor: 'js-repository-static-objects-settings'), html: { class: 'fieldset-form' } do |f|
+ = form_errors(@application_setting)
+
+ %fieldset
+ .form-group
+ = f.label :static_objects_external_storage_url, class: 'label-bold' do
+ = _('External storage URL')
+ = f.text_field :static_objects_external_storage_url, class: 'form-control'
+ %span.form-text.text-muted#static_objects_external_storage_url_help_block
+ = _('URL of the external storage that will serve the repository static objects (e.g. archives, blobs, ...).')
+ .form-group
+ = f.label :static_objects_external_storage_auth_token, class: 'label-bold' do
+ = _('External storage authentication token')
+ = f.text_field :static_objects_external_storage_auth_token, class: 'form-control'
+ %span.form-text.text-muted#static_objects_external_storage_auth_token_help_block
+ = _('A secure token that identifies an external storage request.')
+
+ = f.submit _('Save changes'), class: "btn btn-success"
diff --git a/app/views/admin/application_settings/repository.html.haml b/app/views/admin/application_settings/repository.html.haml
index b50a0dd5a18..25f8b6541b5 100644
--- a/app/views/admin/application_settings/repository.html.haml
+++ b/app/views/admin/application_settings/repository.html.haml
@@ -34,3 +34,14 @@
= _('Configure automatic git checks and housekeeping on repositories.')
.settings-content
= render 'repository_check'
+
+%section.settings.as-repository-static-objects.no-animate#js-repository-static-objects-settings{ class: ('expanded' if expanded_by_default?) }
+ .settings-header
+ %h4
+ = _('Repository static objects')
+ %button.btn.btn-default.js-settings-toggle{ type: 'button' }
+ = expanded_by_default? ? _('Collapse') : _('Expand')
+ %p
+ = _('Serve repository static objects (e.g. archives, blobs, ...) from an external storage (e.g. a CDN).')
+ .settings-content
+ = render 'repository_static_objects'
diff --git a/app/views/profiles/personal_access_tokens/index.html.haml b/app/views/profiles/personal_access_tokens/index.html.haml
index 08a39fc4f58..d9e94908b80 100644
--- a/app/views/profiles/personal_access_tokens/index.html.haml
+++ b/app/views/profiles/personal_access_tokens/index.html.haml
@@ -54,3 +54,23 @@
- reset_link = link_to s_('AccessTokens|reset it'), [:reset, :incoming_email_token, :profile], method: :put, data: { confirm: s_('AccessTokens|Are you sure? Any issue email addresses currently in use will stop working.') }
- reset_message = s_('AccessTokens|Keep this token secret. Anyone who gets ahold of it can create issues as if they were you. You should %{link_reset_it} if that ever happens.') % { link_reset_it: reset_link }
= reset_message.html_safe
+
+- if static_objects_external_storage_enabled?
+ %hr
+ .row.prepend-top-default
+ .col-lg-4
+ %h4.prepend-top-0
+ = s_('AccessTokens|Static object token')
+ %p
+ = s_('AccessTokens|Your static object token is used to authenticate you when repository static objects (e.g. archives, blobs, ...) are being served from an external storage.')
+ %p
+ = s_('AccessTokens|It cannot be used to access any other data.')
+ .col-lg-8
+ = label_tag :static_object_token, s_('AccessTokens|Static object token'), class: "label-bold"
+ = text_field_tag :static_object_token, current_user.static_object_token, class: 'form-control', readonly: true, onclick: 'this.select()'
+ %p.form-text.text-muted
+ - reset_link = url_for [:reset, :static_object_token, :profile]
+ - reset_link_start = '<a data-confirm="%{confirm}" rel="nofollow" data-method="put" href="%{url}">'.html_safe % { confirm: s_('AccessTokens|Are you sure?'), url: reset_link }
+ - reset_link_end = '</a>'.html_safe
+ - reset_message = s_('AccessTokens|Keep this token secret. Anyone who gets ahold of it can access repository static objects as if they were you. You should %{reset_link_start}reset it%{reset_link_end} if that ever happens.') % { reset_link_start: reset_link_start, reset_link_end: reset_link_end }
+ = reset_message.html_safe
diff --git a/app/views/projects/buttons/_download_links.html.haml b/app/views/projects/buttons/_download_links.html.haml
index d344167a6c5..b256d94065b 100644
--- a/app/views/projects/buttons/_download_links.html.haml
+++ b/app/views/projects/buttons/_download_links.html.haml
@@ -2,4 +2,5 @@
.btn-group.ml-0.w-100
- formats.each do |(fmt, extra_class)|
- = link_to fmt, project_archive_path(project, id: tree_join(ref, archive_prefix), path: path, format: fmt), rel: 'nofollow', download: '', class: "btn btn-xs #{extra_class}"
+ - archive_path = project_archive_path(project, id: tree_join(ref, archive_prefix), path: path, format: fmt)
+ = link_to fmt, external_storage_url_or_path(archive_path), rel: 'nofollow', download: '', class: "btn btn-xs #{extra_class}"
diff --git a/changelogs/unreleased/static-objects-external-storage.yml b/changelogs/unreleased/static-objects-external-storage.yml
new file mode 100644
index 00000000000..fd687b2262c
--- /dev/null
+++ b/changelogs/unreleased/static-objects-external-storage.yml
@@ -0,0 +1,5 @@
+---
+title: Enable serving static objects from an external storage
+merge_request: 31025
+author:
+type: added
diff --git a/config/routes/profile.rb b/config/routes/profile.rb
index 83a2b33514b..403f430850e 100644
--- a/config/routes/profile.rb
+++ b/config/routes/profile.rb
@@ -8,6 +8,7 @@ resource :profile, only: [:show, :update] do
put :reset_incoming_email_token
put :reset_feed_token
+ put :reset_static_object_token
put :update_username
end
diff --git a/db/migrate/20190722104947_add_static_object_token_to_users.rb b/db/migrate/20190722104947_add_static_object_token_to_users.rb
new file mode 100644
index 00000000000..6ef85d9acaa
--- /dev/null
+++ b/db/migrate/20190722104947_add_static_object_token_to_users.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddStaticObjectTokenToUsers < ActiveRecord::Migration[5.2]
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ add_column :users, :static_object_token, :string, limit: 255
+ end
+
+ def down
+ remove_column :users, :static_object_token
+ end
+end
diff --git a/db/migrate/20190722132830_add_static_objects_external_storage_columns_to_application_settings.rb b/db/migrate/20190722132830_add_static_objects_external_storage_columns_to_application_settings.rb
new file mode 100644
index 00000000000..a23e6ed66cd
--- /dev/null
+++ b/db/migrate/20190722132830_add_static_objects_external_storage_columns_to_application_settings.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddStaticObjectsExternalStorageColumnsToApplicationSettings < ActiveRecord::Migration[5.2]
+ # Set this constant to true if this migration requires downtime.
+ DOWNTIME = false
+
+ def change
+ add_column :application_settings, :static_objects_external_storage_url, :string, limit: 255
+ add_column :application_settings, :static_objects_external_storage_auth_token, :string, limit: 255
+ end
+end
diff --git a/db/migrate/20190725183432_add_index_to_index_on_static_object_token.rb b/db/migrate/20190725183432_add_index_to_index_on_static_object_token.rb
new file mode 100644
index 00000000000..423c45b9543
--- /dev/null
+++ b/db/migrate/20190725183432_add_index_to_index_on_static_object_token.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddIndexToIndexOnStaticObjectToken < ActiveRecord::Migration[5.2]
+ include Gitlab::Database::MigrationHelpers
+
+ # Set this constant to true if this migration requires downtime.
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ add_concurrent_index :users, :static_object_token, unique: true
+ end
+
+ def down
+ remove_concurrent_index :users, :static_object_token
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 61f7787f192..d4db1db5c3f 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -284,6 +284,8 @@ ActiveRecord::Schema.define(version: 2019_09_05_223900) do
t.text "asset_proxy_whitelist"
t.text "encrypted_asset_proxy_secret_key"
t.string "encrypted_asset_proxy_secret_key_iv"
+ t.string "static_objects_external_storage_url", limit: 255
+ t.string "static_objects_external_storage_auth_token", limit: 255
t.index ["custom_project_templates_group_id"], name: "index_application_settings_on_custom_project_templates_group_id"
t.index ["file_template_project_id"], name: "index_application_settings_on_file_template_project_id"
t.index ["instance_administration_project_id"], name: "index_applicationsettings_on_instance_administration_project_id"
@@ -3548,6 +3550,7 @@ ActiveRecord::Schema.define(version: 2019_09_05_223900) do
t.integer "bot_type", limit: 2
t.string "first_name", limit: 255
t.string "last_name", limit: 255
+ t.string "static_object_token", limit: 255
t.index ["accepted_term_id"], name: "index_users_on_accepted_term_id"
t.index ["admin"], name: "index_users_on_admin"
t.index ["bot_type"], name: "index_users_on_bot_type"
@@ -3567,6 +3570,7 @@ ActiveRecord::Schema.define(version: 2019_09_05_223900) do
t.index ["state"], name: "index_users_on_state"
t.index ["state"], name: "index_users_on_state_and_internal", where: "(ghost IS NOT TRUE)"
t.index ["state"], name: "index_users_on_state_and_internal_ee", where: "((ghost IS NOT TRUE) AND (bot_type IS NULL))"
+ t.index ["static_object_token"], name: "index_users_on_static_object_token", unique: true
t.index ["unconfirmed_email"], name: "index_users_on_unconfirmed_email", where: "(unconfirmed_email IS NOT NULL)"
t.index ["username"], name: "index_users_on_username"
t.index ["username"], name: "index_users_on_username_trigram", opclass: :gin_trgm_ops, using: :gin
diff --git a/doc/administration/index.md b/doc/administration/index.md
index d557068e6c8..ae38834b68c 100644
--- a/doc/administration/index.md
+++ b/doc/administration/index.md
@@ -142,6 +142,7 @@ Learn how to install, configure, update, and maintain your GitLab instance.
- [Repository storage types](repository_storage_types.md): Information about the different repository storage types.
- [Repository storage rake tasks](raketasks/storage.md): A collection of rake tasks to list and migrate existing projects and attachments associated with it from Legacy storage to Hashed storage.
- [Limit repository size](../user/admin_area/settings/account_and_limit_settings.md): Set a hard limit for your repositories' size. **(STARTER ONLY)**
+- [Static objects external storage](static_objects_external_storage.md): Set external storage for static objects in a repository.
## Continuous Integration settings
diff --git a/doc/administration/static_objects_external_storage.md b/doc/administration/static_objects_external_storage.md
new file mode 100644
index 00000000000..e4d60c77199
--- /dev/null
+++ b/doc/administration/static_objects_external_storage.md
@@ -0,0 +1,50 @@
+# Static objects external storage
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/31025) in GitLab 12.3.
+
+GitLab can be configured to serve repository static objects (for example, archives) from an external
+storage, such as a CDN.
+
+## Configuring
+
+To configure external storage for static objects:
+
+1. Navigate to **Admin Area > Settings > Repository**.
+1. Expand the **Repository static objects** section.
+1. Enter the base URL and an arbitrary token.
+
+The token is required to distinguish requests coming from the external storage, so users don't
+circumvent the external storage and go for the application directly. The token is expected to be
+set in the `X-Gitlab-External-Storage-Token` header in requests originating from the external
+storage.
+
+## Serving private static objects
+
+GitLab will append a user-specific token for static object URLs that belong to private projects,
+so an external storage can be authenticated on behalf of the user. When processing requests originating
+from the external storage, GitLab will look for the token in the `token` query parameter or in
+the `X-Gitlab-Static-Object-Token` header to check the user's ability to access the requested object.
+
+## Requests flow example
+
+The following example shows a sequence of requests and responses between the user,
+GitLab, and the CDN:
+
+```mermaid
+sequenceDiagram
+ User->>GitLab: GET /project/-/archive/master.zip
+ GitLab->>User: 302 Found
+ Note over User,GitLab: Location: https://cdn.com/project/-/archive/master.zip?token=secure-user-token
+ User->>CDN: GET /project/-/archive/master.zip?token=secure-user-token
+ alt object not in cache
+ CDN->>GitLab: GET /project/-/archive/master.zip
+ Note over CDN,GitLab: X-Gitlab-External-Storage-Token: secure-cdn-token<br/>X-Gitlab-Static-Object-Token: secure-user-token
+ GitLab->>CDN: 200 OK
+ CDN->>User: master.zip
+ else object in cache
+ CDN->>GitLab: GET /project/-/archive/master.zip
+ Note over CDN,GitLab: X-Gitlab-External-Storage-Token: secure-cdn-token<br/>X-Gitlab-Static-Object-Token: secure-user-token<br/>If-None-Match: etag-value
+ GitLab->>CDN: 304 Not Modified
+ CDN->>User: master.zip
+ end
+```
diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb
index 176766d1a8b..aca8804b04c 100644
--- a/lib/gitlab/auth/request_authenticator.rb
+++ b/lib/gitlab/auth/request_authenticator.rb
@@ -24,7 +24,9 @@ module Gitlab
end
def find_sessionless_user(request_format)
- find_user_from_web_access_token(request_format) || find_user_from_feed_token(request_format)
+ find_user_from_web_access_token(request_format) ||
+ find_user_from_feed_token(request_format) ||
+ find_user_from_static_object_token(request_format)
rescue Gitlab::Auth::AuthenticationError
nil
end
diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb
index 97755117edc..76d41eede23 100644
--- a/lib/gitlab/auth/user_auth_finders.rb
+++ b/lib/gitlab/auth/user_auth_finders.rb
@@ -28,6 +28,15 @@ module Gitlab
current_request.env['warden']&.authenticate if verified_request?
end
+ def find_user_from_static_object_token(request_format)
+ return unless valid_static_objects_format?(request_format)
+
+ token = current_request.params[:token].presence || current_request.headers['X-Gitlab-Static-Object-Token'].presence
+ return unless token
+
+ User.find_by_static_object_token(token) || raise(UnauthorizedError)
+ end
+
def find_user_from_feed_token(request_format)
return unless valid_rss_format?(request_format)
@@ -154,6 +163,15 @@ module Gitlab
end
end
+ def valid_static_objects_format?(request_format)
+ case request_format
+ when :archive
+ archive_request?
+ else
+ false
+ end
+ end
+
def rss_request?
current_request.path.ends_with?('.atom') || current_request.format.atom?
end
@@ -165,6 +183,10 @@ module Gitlab
def api_request?
current_request.path.starts_with?("/api/")
end
+
+ def archive_request?
+ current_request.path.include?('/-/archive/')
+ end
end
end
end
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index e15000b5184..3f525529e0b 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -517,6 +517,9 @@ msgstr ""
msgid "A regular expression that will be used to find the test coverage output in the job trace. Leave blank to disable"
msgstr ""
+msgid "A secure token that identifies an external storage request."
+msgstr ""
+
msgid "A user with write access to the source branch selected this option"
msgstr ""
@@ -568,6 +571,9 @@ msgstr ""
msgid "AccessTokens|Access Tokens"
msgstr ""
+msgid "AccessTokens|Are you sure?"
+msgstr ""
+
msgid "AccessTokens|Are you sure? Any RSS or calendar URLs currently in use will stop working."
msgstr ""
@@ -586,6 +592,9 @@ msgstr ""
msgid "AccessTokens|It cannot be used to access any other data."
msgstr ""
+msgid "AccessTokens|Keep this token secret. Anyone who gets ahold of it can access repository static objects as if they were you. You should %{reset_link_start}reset it%{reset_link_end} if that ever happens."
+msgstr ""
+
msgid "AccessTokens|Keep this token secret. Anyone who gets ahold of it can create issues as if they were you. You should %{link_reset_it} if that ever happens."
msgstr ""
@@ -595,6 +604,9 @@ msgstr ""
msgid "AccessTokens|Personal Access Tokens"
msgstr ""
+msgid "AccessTokens|Static object token"
+msgstr ""
+
msgid "AccessTokens|They are the only accepted password when you have Two-Factor Authentication (2FA) enabled."
msgstr ""
@@ -610,6 +622,9 @@ msgstr ""
msgid "AccessTokens|Your incoming email token is used to authenticate you when you create a new issue by email, and is included in your personal project-specific email addresses."
msgstr ""
+msgid "AccessTokens|Your static object token is used to authenticate you when repository static objects (e.g. archives, blobs, ...) are being served from an external storage."
+msgstr ""
+
msgid "AccessTokens|reset it"
msgstr ""
@@ -4852,6 +4867,12 @@ msgstr ""
msgid "External authorization request timeout"
msgstr ""
+msgid "External storage URL"
+msgstr ""
+
+msgid "External storage authentication token"
+msgstr ""
+
msgid "ExternalAuthorizationService|Classification label"
msgstr ""
@@ -8668,6 +8689,9 @@ msgstr ""
msgid "Profiles|Some options are unavailable for LDAP accounts"
msgstr ""
+msgid "Profiles|Static object token was successfully reset"
+msgstr ""
+
msgid "Profiles|Tell us about yourself in fewer than 250 characters"
msgstr ""
@@ -9693,6 +9717,9 @@ msgstr ""
msgid "Repository mirror"
msgstr ""
+msgid "Repository static objects"
+msgstr ""
+
msgid "Repository storage"
msgstr ""
@@ -10244,6 +10271,9 @@ msgstr ""
msgid "September"
msgstr ""
+msgid "Serve repository static objects (e.g. archives, blobs, ...) from an external storage (e.g. a CDN)."
+msgstr ""
+
msgid "Server supports batch API only, please update your Git LFS client to version 1.0.1 and up."
msgstr ""
@@ -12421,6 +12451,9 @@ msgstr ""
msgid "U2F only works with HTTPS-enabled websites. Contact your administrator for more details."
msgstr ""
+msgid "URL of the external storage that will serve the repository static objects (e.g. archives, blobs, ...)."
+msgstr ""
+
msgid "Unable to apply suggestions to a deleted line."
msgstr ""
diff --git a/spec/controllers/concerns/static_object_external_storage_spec.rb b/spec/controllers/concerns/static_object_external_storage_spec.rb
new file mode 100644
index 00000000000..3a0219ddaa1
--- /dev/null
+++ b/spec/controllers/concerns/static_object_external_storage_spec.rb
@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe StaticObjectExternalStorage do
+ controller(Projects::ApplicationController) do
+ include StaticObjectExternalStorage # rubocop:disable RSpec/DescribedClass
+
+ before_action :redirect_to_external_storage, if: :static_objects_external_storage_enabled?
+
+ def show
+ head :ok
+ end
+ end
+
+ let(:project) { create(:project, :public) }
+ let(:user) { create(:user, static_object_token: 'hunter1') }
+
+ before do
+ project.add_developer(user)
+ sign_in(user)
+ end
+
+ context 'when external storage is not configured' do
+ it 'calls the action normally' do
+ expect(Gitlab::CurrentSettings.static_objects_external_storage_url).to be_blank
+
+ do_request
+
+ expect(response).to have_gitlab_http_status(200)
+ end
+ end
+
+ context 'when external storage is configured' do
+ before do
+ allow_any_instance_of(ApplicationSetting).to receive(:static_objects_external_storage_url).and_return('https://cdn.gitlab.com')
+ allow_any_instance_of(ApplicationSetting).to receive(:static_objects_external_storage_auth_token).and_return('letmein')
+
+ routes.draw { get '/:namespace_id/:id' => 'projects/application#show' }
+ end
+
+ context 'when external storage token is empty' do
+ let(:base_redirect_url) { "https://cdn.gitlab.com/#{project.namespace.to_param}/#{project.to_param}" }
+
+ context 'when project is public' do
+ it 'redirects to external storage URL without adding a token parameter' do
+ do_request
+
+ expect(response).to redirect_to(base_redirect_url)
+ end
+ end
+
+ context 'when project is not public' do
+ let(:project) { create(:project, :private) }
+
+ it 'redirects to external storage URL a token parameter added' do
+ do_request
+
+ expect(response).to redirect_to("#{base_redirect_url}?token=#{user.static_object_token}")
+ end
+
+ context 'when path includes extra parameters' do
+ it 'includes the parameters in the redirect URL' do
+ do_request(foo: 'bar')
+
+ expect(response.location).to eq("#{base_redirect_url}?foo=bar&token=#{user.static_object_token}")
+ end
+ end
+ end
+ end
+
+ context 'when external storage token is present' do
+ context 'when token is correct' do
+ it 'calls the action normally' do
+ request.headers['X-Gitlab-External-Storage-Token'] = 'letmein'
+ do_request
+
+ expect(response).to have_gitlab_http_status(200)
+ end
+ end
+
+ context 'when token is incorrect' do
+ it 'return 403' do
+ request.headers['X-Gitlab-External-Storage-Token'] = 'donotletmein'
+ do_request
+
+ expect(response).to have_gitlab_http_status(403)
+ end
+ end
+ end
+ end
+
+ def do_request(extra_params = {})
+ get :show, params: { namespace_id: project.namespace, id: project }.merge(extra_params)
+ end
+end
diff --git a/spec/controllers/projects/repositories_controller_spec.rb b/spec/controllers/projects/repositories_controller_spec.rb
index fcab4d73dca..084644484c5 100644
--- a/spec/controllers/projects/repositories_controller_spec.rb
+++ b/spec/controllers/projects/repositories_controller_spec.rb
@@ -125,5 +125,59 @@ describe Projects::RepositoriesController do
end
end
end
+
+ context 'as a sessionless user' do
+ let(:user) { create(:user) }
+
+ before do
+ project.add_developer(user)
+ end
+
+ context 'when no token is provided' do
+ it 'redirects to sign in page' do
+ get :archive, params: { namespace_id: project.namespace, project_id: project, id: 'master' }, format: 'zip'
+
+ expect(response).to have_gitlab_http_status(302)
+ end
+ end
+
+ context 'when a token param is present' do
+ context 'when token is correct' do
+ it 'calls the action normally' do
+ get :archive, params: { namespace_id: project.namespace, project_id: project, id: 'master', token: user.static_object_token }, format: 'zip'
+
+ expect(response).to have_gitlab_http_status(200)
+ end
+ end
+
+ context 'when token is incorrect' do
+ it 'redirects to sign in page' do
+ get :archive, params: { namespace_id: project.namespace, project_id: project, id: 'master', token: 'foobar' }, format: 'zip'
+
+ expect(response).to have_gitlab_http_status(302)
+ end
+ end
+ end
+
+ context 'when a token header is present' do
+ context 'when token is correct' do
+ it 'calls the action normally' do
+ request.headers['X-Gitlab-Static-Object-Token'] = user.static_object_token
+ get :archive, params: { namespace_id: project.namespace, project_id: project, id: 'master' }, format: 'zip'
+
+ expect(response).to have_gitlab_http_status(200)
+ end
+ end
+
+ context 'when token is incorrect' do
+ it 'redirects to sign in page' do
+ request.headers['X-Gitlab-Static-Object-Token'] = 'foobar'
+ get :archive, params: { namespace_id: project.namespace, project_id: project, id: 'master' }, format: 'zip'
+
+ expect(response).to have_gitlab_http_status(302)
+ end
+ end
+ end
+ end
end
end
diff --git a/spec/features/projects/branches/download_buttons_spec.rb b/spec/features/projects/branches/download_buttons_spec.rb
index 401425187b0..e0b0e22823e 100644
--- a/spec/features/projects/branches/download_buttons_spec.rb
+++ b/spec/features/projects/branches/download_buttons_spec.rb
@@ -29,6 +29,11 @@ describe 'Download buttons in branches page' do
end
describe 'when checking branches' do
+ it_behaves_like 'archive download buttons' do
+ let(:ref) { 'binary-encoding' }
+ let(:path_to_visit) { project_branches_filtered_path(project, state: 'all', search: ref) }
+ end
+
context 'with artifacts' do
before do
visit project_branches_filtered_path(project, state: 'all', search: 'binary-encoding')
diff --git a/spec/features/projects/files/download_buttons_spec.rb b/spec/features/projects/files/download_buttons_spec.rb
index a4889f8d4c4..871f5212ddd 100644
--- a/spec/features/projects/files/download_buttons_spec.rb
+++ b/spec/features/projects/files/download_buttons_spec.rb
@@ -24,11 +24,17 @@ describe 'Projects > Files > Download buttons in files tree' do
before do
sign_in(user)
project.add_developer(user)
+ end
- visit project_tree_path(project, project.default_branch)
+ it_behaves_like 'archive download buttons' do
+ let(:path_to_visit) { project_tree_path(project, project.default_branch) }
end
context 'with artifacts' do
+ before do
+ visit project_tree_path(project, project.default_branch)
+ end
+
it 'shows download artifacts button' do
href = latest_succeeded_project_artifacts_path(project, "#{project.default_branch}/download", job: 'build')
diff --git a/spec/features/projects/show/download_buttons_spec.rb b/spec/features/projects/show/download_buttons_spec.rb
index 5e7453bcdb7..0d609069426 100644
--- a/spec/features/projects/show/download_buttons_spec.rb
+++ b/spec/features/projects/show/download_buttons_spec.rb
@@ -29,6 +29,8 @@ describe 'Projects > Show > Download buttons' do
end
describe 'when checking project main page' do
+ it_behaves_like 'archive download buttons'
+
context 'with artifacts' do
before do
visit project_path(project)
diff --git a/spec/features/projects/tags/download_buttons_spec.rb b/spec/features/projects/tags/download_buttons_spec.rb
index 76b2704ae49..64141cf5dc9 100644
--- a/spec/features/projects/tags/download_buttons_spec.rb
+++ b/spec/features/projects/tags/download_buttons_spec.rb
@@ -30,6 +30,11 @@ describe 'Download buttons in tags page' do
end
describe 'when checking tags' do
+ it_behaves_like 'archive download buttons' do
+ let(:path_to_visit) { project_tags_path(project) }
+ let(:ref) { tag }
+ end
+
context 'with artifacts' do
before do
visit project_tags_path(project)
diff --git a/spec/helpers/application_helper_spec.rb b/spec/helpers/application_helper_spec.rb
index b81249a1e29..4a3ff7e0095 100644
--- a/spec/helpers/application_helper_spec.rb
+++ b/spec/helpers/application_helper_spec.rb
@@ -195,4 +195,41 @@ describe ApplicationHelper do
end
end
end
+
+ describe '#external_storage_url_or_path' do
+ let(:project) { create(:project) }
+
+ context 'when external storage is disabled' do
+ it 'returns the passed path' do
+ expect(helper.external_storage_url_or_path('/foo/bar', project)).to eq('/foo/bar')
+ end
+ end
+
+ context 'when external storage is enabled' do
+ let(:user) { create(:user, static_object_token: 'hunter1') }
+
+ before do
+ allow_any_instance_of(ApplicationSetting).to receive(:static_objects_external_storage_url).and_return('https://cdn.gitlab.com')
+ allow(helper).to receive(:current_user).and_return(user)
+ end
+
+ it 'returns the external storage URL prepended to the path' do
+ expect(helper.external_storage_url_or_path('/foo/bar', project)).to eq("https://cdn.gitlab.com/foo/bar?token=#{user.static_object_token}")
+ end
+
+ it 'preserves the path query parameters' do
+ url = helper.external_storage_url_or_path('/foo/bar?unicode=1', project)
+
+ expect(url).to eq("https://cdn.gitlab.com/foo/bar?token=#{user.static_object_token}&unicode=1")
+ end
+
+ context 'when project is public' do
+ let(:project) { create(:project, :public) }
+
+ it 'returns does not append a token parameter' do
+ expect(helper.external_storage_url_or_path('/foo/bar', project)).to eq('https://cdn.gitlab.com/foo/bar')
+ end
+ end
+ end
+ end
end
diff --git a/spec/lib/gitlab/auth/user_auth_finders_spec.rb b/spec/lib/gitlab/auth/user_auth_finders_spec.rb
index 41265da97a4..dd8070c1240 100644
--- a/spec/lib/gitlab/auth/user_auth_finders_spec.rb
+++ b/spec/lib/gitlab/auth/user_auth_finders_spec.rb
@@ -115,6 +115,60 @@ describe Gitlab::Auth::UserAuthFinders do
end
end
+ describe '#find_user_from_static_object_token' do
+ context 'when request format is archive' do
+ before do
+ env['SCRIPT_NAME'] = 'project/-/archive/master.zip'
+ end
+
+ context 'when token header param is present' do
+ context 'when token is correct' do
+ it 'returns the user' do
+ request.headers['X-Gitlab-Static-Object-Token'] = user.static_object_token
+
+ expect(find_user_from_static_object_token(:archive)).to eq(user)
+ end
+ end
+
+ context 'when token is incorrect' do
+ it 'returns the user' do
+ request.headers['X-Gitlab-Static-Object-Token'] = 'foobar'
+
+ expect { find_user_from_static_object_token(:archive) }.to raise_error(Gitlab::Auth::UnauthorizedError)
+ end
+ end
+ end
+
+ context 'when token query param is present' do
+ context 'when token is correct' do
+ it 'returns the user' do
+ set_param(:token, user.static_object_token)
+
+ expect(find_user_from_static_object_token(:archive)).to eq(user)
+ end
+ end
+
+ context 'when token is incorrect' do
+ it 'returns the user' do
+ set_param(:token, 'foobar')
+
+ expect { find_user_from_static_object_token(:archive) }.to raise_error(Gitlab::Auth::UnauthorizedError)
+ end
+ end
+ end
+ end
+
+ context 'when request format is not archive' do
+ before do
+ env['script_name'] = 'url'
+ end
+
+ it 'returns nil' do
+ expect(find_user_from_static_object_token(:foo)).to be_nil
+ end
+ end
+ end
+
describe '#find_user_from_access_token' do
let(:personal_access_token) { create(:personal_access_token, user: user) }
diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb
index 4f7a6d102b8..d12f9b9100a 100644
--- a/spec/models/application_setting_spec.rb
+++ b/spec/models/application_setting_spec.rb
@@ -48,6 +48,10 @@ describe ApplicationSetting do
it { is_expected.not_to allow_value(nil).for(:outbound_local_requests_whitelist) }
it { is_expected.to allow_value([]).for(:outbound_local_requests_whitelist) }
+ it { is_expected.to allow_value(nil).for(:static_objects_external_storage_url) }
+ it { is_expected.to allow_value(http).for(:static_objects_external_storage_url) }
+ it { is_expected.to allow_value(https).for(:static_objects_external_storage_url) }
+
context "when user accepted let's encrypt terms of service" do
before do
setting.update(lets_encrypt_terms_of_service_accepted: true)
@@ -420,6 +424,16 @@ describe ApplicationSetting do
end
end
end
+
+ context 'static objects external storage' do
+ context 'when URL is set' do
+ before do
+ subject.static_objects_external_storage_url = http
+ end
+
+ it { is_expected.not_to allow_value(nil).for(:static_objects_external_storage_auth_token) }
+ end
+ end
end
context 'restrict creating duplicates' do
diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb
index b8c323904b8..741563292e9 100644
--- a/spec/models/user_spec.rb
+++ b/spec/models/user_spec.rb
@@ -945,6 +945,16 @@ describe User do
end
end
+ describe 'static object token' do
+ it 'ensures a static object token on read' do
+ user = create(:user, static_object_token: nil)
+ static_object_token = user.static_object_token
+
+ expect(static_object_token).not_to be_blank
+ expect(user.reload.static_object_token).to eq static_object_token
+ end
+ end
+
describe '#recently_sent_password_reset?' do
it 'is false when reset_password_sent_at is nil' do
user = build_stubbed(:user, reset_password_sent_at: nil)
diff --git a/spec/support/shared_examples/features/archive_download_buttons_shared_examples.rb b/spec/support/shared_examples/features/archive_download_buttons_shared_examples.rb
new file mode 100644
index 00000000000..920fcbde483
--- /dev/null
+++ b/spec/support/shared_examples/features/archive_download_buttons_shared_examples.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+shared_examples 'archive download buttons' do
+ let(:formats) { %w(zip tar.gz tar.bz2 tar) }
+ let(:path_to_visit) { project_path(project) }
+ let(:ref) { project.default_branch }
+
+ context 'when static objects external storage is enabled' do
+ before do
+ allow_any_instance_of(ApplicationSetting).to receive(:static_objects_external_storage_url).and_return('https://cdn.gitlab.com')
+ visit path_to_visit
+ end
+
+ context 'private project' do
+ it 'shows archive download buttons with external storage URL prepended and user token appended to their href' do
+ formats.each do |format|
+ path = archive_path(project, ref, format)
+ uri = URI('https://cdn.gitlab.com')
+ uri.path = path
+ uri.query = "token=#{user.static_object_token}"
+
+ expect(page).to have_link format, href: uri.to_s
+ end
+ end
+ end
+
+ context 'public project' do
+ let(:project) { create(:project, :repository, :public) }
+
+ it 'shows archive download buttons with external storage URL prepended to their href' do
+ formats.each do |format|
+ path = archive_path(project, ref, format)
+ uri = URI('https://cdn.gitlab.com')
+ uri.path = path
+
+ expect(page).to have_link format, href: uri.to_s
+ end
+ end
+ end
+ end
+
+ context 'when static objects external storage is disabled' do
+ before do
+ visit path_to_visit
+ end
+
+ it 'shows default archive download buttons' do
+ formats.each do |format|
+ path = archive_path(project, ref, format)
+
+ expect(page).to have_link format, href: path
+ end
+ end
+ end
+
+ def archive_path(project, ref, format)
+ project_archive_path(project, id: "#{ref}/#{project.path}-#{ref}", path: nil, format: format)
+ end
+end