Enable serving static objects from an external storagestatic-objects-external-storage

It consists of two parts:

1. Redirecting users to the configured external storage
1. Allowing the external storage to request the static object(s)
   on behalf of the user by means of specific tokens

Part of https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/6829

13 files changed, 131 insertions, 3 deletions

diff --git a/app/controllers/concerns/sessionless_authentication.rb b/app/controllers/concerns/sessionless_authentication.rb
index 4304b8565ce..ba06384a37a 100644
--- a/app/controllers/concerns/sessionless_authentication.rb
+++ b/app/controllers/concerns/sessionless_authentication.rb
@@ -2,10 +2,10 @@
 
 # == SessionlessAuthentication
 #
-# Controller concern to handle PAT and RSS token authentication methods
+# Controller concern to handle PAT, RSS, and static objects token authentication methods
 #
 module SessionlessAuthentication
-  # This filter handles personal access tokens, and atom requests with rss tokens
+  # This filter handles personal access tokens, atom requests with rss tokens, and static object tokens
   def authenticate_sessionless_user!(request_format)
     user = Gitlab::Auth::RequestAuthenticator.new(request).find_sessionless_user(request_format)
 
diff --git a/app/controllers/concerns/static_object_external_storage.rb b/app/controllers/concerns/static_object_external_storage.rb
new file mode 100644
index 00000000000..dbfe0ed3adf
--- /dev/null
+++ b/app/controllers/concerns/static_object_external_storage.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module StaticObjectExternalStorage
+  extend ActiveSupport::Concern
+
+  included do
+    include ApplicationHelper
+  end
+
+  def redirect_to_external_storage
+    return if external_storage_request?
+
+    redirect_to external_storage_url_or_path(request.fullpath, project)
+  end
+
+  def external_storage_request?
+    header_token = request.headers['X-Gitlab-External-Storage-Token']
+    return false unless header_token.present?
+
+    external_storage_token = Gitlab::CurrentSettings.static_objects_external_storage_auth_token
+    ActiveSupport::SecurityUtils.secure_compare(header_token, external_storage_token) ||
+      raise(Gitlab::Access::AccessDeniedError)
+  end
+end
diff --git a/app/controllers/profiles_controller.rb b/app/controllers/profiles_controller.rb
index 1d16ddb1608..958a24b6c0e 100644
--- a/app/controllers/profiles_controller.rb
+++ b/app/controllers/profiles_controller.rb
@@ -46,6 +46,15 @@ class ProfilesController < Profiles::ApplicationController
     redirect_to profile_personal_access_tokens_path
   end
 
+  def reset_static_object_token
+    Users::UpdateService.new(current_user, user: @user).execute! do |user|
+      user.reset_static_object_token!
+    end
+
+    redirect_to profile_personal_access_tokens_path,
+      notice: s_('Profiles|Static object token was successfully reset')
+  end
+
   # rubocop: disable CodeReuse/ActiveRecord
   def audit_log
     @events = AuditEvent.where(entity_type: "User", entity_id: current_user.id)
diff --git a/app/controllers/projects/repositories_controller.rb b/app/controllers/projects/repositories_controller.rb
index a51759641e4..d69f9e65874 100644
--- a/app/controllers/projects/repositories_controller.rb
+++ b/app/controllers/projects/repositories_controller.rb
@@ -2,6 +2,9 @@
 
 class Projects::RepositoriesController < Projects::ApplicationController
   include ExtractsPath
+  include StaticObjectExternalStorage
+
+  prepend_before_action(only: [:archive]) { authenticate_sessionless_user!(:archive) }
 
   # Authorize
   before_action :require_non_empty_project, except: :create
@@ -9,6 +12,7 @@ class Projects::RepositoriesController < Projects::ApplicationController
   before_action :assign_append_sha, only: :archive
   before_action :authorize_download_code!
   before_action :authorize_admin_project!, only: :create
+  before_action :redirect_to_external_storage, only: :archive, if: :static_objects_external_storage_enabled?
 
   def create
     @project.create_repository
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
index ffa5719fefb..1671aa5bd04 100644
--- a/app/helpers/application_helper.rb
+++ b/app/helpers/application_helper.rb
@@ -169,6 +169,25 @@ module ApplicationHelper
     Gitlab::CurrentSettings.current_application_settings.help_page_support_url.presence || promo_url + '/getting-help/'
   end
 
+  def static_objects_external_storage_enabled?
+    Gitlab::CurrentSettings.static_objects_external_storage_enabled?
+  end
+
+  def external_storage_url_or_path(path, project = @project)
+    return path unless static_objects_external_storage_enabled?
+
+    uri = URI(Gitlab::CurrentSettings.static_objects_external_storage_url)
+    path = URI(path) # `path` could have query parameters, so we need to split query and path apart
+
+    query = Rack::Utils.parse_nested_query(path.query)
+    query['token'] = current_user.static_object_token unless project.public?
+
+    uri.path = path.path
+    uri.query = query.to_query unless query.empty?
+
+    uri.to_s
+  end
+
   def page_filter_path(options = {})
     without = options.delete(:without)
 
diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb
index 84021d0da56..be3b67c60e8 100644
--- a/app/helpers/application_settings_helper.rb
+++ b/app/helpers/application_settings_helper.rb
@@ -168,6 +168,8 @@ module ApplicationSettingsHelper
       :asset_proxy_secret_key,
       :asset_proxy_url,
       :asset_proxy_whitelist,
+      :static_objects_external_storage_auth_token,
+      :static_objects_external_storage_url,
       :authorized_keys_enabled,
       :auto_devops_enabled,
       :auto_devops_domain,
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index e39d655325f..3409411c3b1 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -8,6 +8,7 @@ class ApplicationSetting < ApplicationRecord
 
   add_authentication_token_field :runners_registration_token, encrypted: -> { Feature.enabled?(:application_settings_tokens_optional_encryption, default_enabled: true) ? :optional : :required }
   add_authentication_token_field :health_check_access_token
+  add_authentication_token_field :static_objects_external_storage_auth_token
 
   belongs_to :instance_administration_project, class_name: "Project"
 
@@ -211,6 +212,13 @@ class ApplicationSetting < ApplicationRecord
             allow_blank: false,
             if: :asset_proxy_enabled?
 
+  validates :static_objects_external_storage_url,
+            addressable_url: true, allow_blank: true
+
+  validates :static_objects_external_storage_auth_token,
+            presence: true,
+            if: :static_objects_external_storage_url?
+
   SUPPORTED_KEY_TYPES.each do |type|
     validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type }
   end
diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb
index f402c0e2775..8d9597aa5a4 100644
--- a/app/models/application_setting_implementation.rb
+++ b/app/models/application_setting_implementation.rb
@@ -306,6 +306,10 @@ module ApplicationSettingImplementation
     archive_builds_in_seconds.seconds.ago if archive_builds_in_seconds
   end
 
+  def static_objects_external_storage_enabled?
+    static_objects_external_storage_url.present?
+  end
+
   private
 
   def array_to_string(arr)
diff --git a/app/models/user.rb b/app/models/user.rb
index 67d730e2fa3..75532aeebb3 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -31,6 +31,7 @@ class User < ApplicationRecord
 
   add_authentication_token_field :incoming_email_token, token_generator: -> { SecureRandom.hex.to_i(16).to_s(36) }
   add_authentication_token_field :feed_token
+  add_authentication_token_field :static_object_token
 
   default_value_for :admin, false
   default_value_for(:external) { Gitlab::CurrentSettings.user_default_external }
@@ -1437,6 +1438,13 @@ class User < ApplicationRecord
     ensure_feed_token!
   end
 
+  # Each existing user needs to have a `static_object_token`.
+  # We do this on read since migrating all existing users is not a feasible
+  # solution.
+  def static_object_token
+    ensure_static_object_token!
+  end
+
   def sync_attribute?(attribute)
     return true if ldap_user? && attribute == :email
 
diff --git a/app/views/admin/application_settings/_repository_static_objects.html.haml b/app/views/admin/application_settings/_repository_static_objects.html.haml
new file mode 100644
index 00000000000..03aa48b2282
--- /dev/null
+++ b/app/views/admin/application_settings/_repository_static_objects.html.haml
@@ -0,0 +1,18 @@
+= form_for @application_setting, url: repository_admin_application_settings_path(anchor: 'js-repository-static-objects-settings'), html: { class: 'fieldset-form' } do |f|
+  = form_errors(@application_setting)
+
+  %fieldset
+    .form-group
+      = f.label :static_objects_external_storage_url, class: 'label-bold' do
+        = _('External storage URL')
+      = f.text_field :static_objects_external_storage_url, class: 'form-control'
+      %span.form-text.text-muted#static_objects_external_storage_url_help_block
+        = _('URL of the external storage that will serve the repository static objects (e.g. archives, blobs, ...).')
+    .form-group
+      = f.label :static_objects_external_storage_auth_token, class: 'label-bold' do
+        = _('External storage authentication token')
+      = f.text_field :static_objects_external_storage_auth_token, class: 'form-control'
+      %span.form-text.text-muted#static_objects_external_storage_auth_token_help_block
+        = _('A secure token that identifies an external storage request.')
+
+  = f.submit _('Save changes'), class: "btn btn-success"
diff --git a/app/views/admin/application_settings/repository.html.haml b/app/views/admin/application_settings/repository.html.haml
index b50a0dd5a18..25f8b6541b5 100644
--- a/app/views/admin/application_settings/repository.html.haml
+++ b/app/views/admin/application_settings/repository.html.haml
@@ -34,3 +34,14 @@
       = _('Configure automatic git checks and housekeeping on repositories.')
   .settings-content
     = render 'repository_check'
+
+%section.settings.as-repository-static-objects.no-animate#js-repository-static-objects-settings{ class: ('expanded' if expanded_by_default?) }
+  .settings-header
+    %h4
+      = _('Repository static objects')
+    %button.btn.btn-default.js-settings-toggle{ type: 'button' }
+      = expanded_by_default? ? _('Collapse') : _('Expand')
+    %p
+      = _('Serve repository static objects (e.g. archives, blobs, ...) from an external storage (e.g. a CDN).')
+  .settings-content
+    = render 'repository_static_objects'
diff --git a/app/views/profiles/personal_access_tokens/index.html.haml b/app/views/profiles/personal_access_tokens/index.html.haml
index 08a39fc4f58..d9e94908b80 100644
--- a/app/views/profiles/personal_access_tokens/index.html.haml
+++ b/app/views/profiles/personal_access_tokens/index.html.haml
@@ -54,3 +54,23 @@
         - reset_link = link_to s_('AccessTokens|reset it'), [:reset, :incoming_email_token, :profile], method: :put, data: { confirm: s_('AccessTokens|Are you sure? Any issue email addresses currently in use will stop working.') }
         - reset_message = s_('AccessTokens|Keep this token secret. Anyone who gets ahold of it can create issues as if they were you. You should %{link_reset_it} if that ever happens.') % { link_reset_it: reset_link }
         = reset_message.html_safe
+
+- if static_objects_external_storage_enabled?
+  %hr
+  .row.prepend-top-default
+    .col-lg-4
+      %h4.prepend-top-0
+        = s_('AccessTokens|Static object token')
+      %p
+        = s_('AccessTokens|Your static object token is used to authenticate you when repository static objects (e.g. archives, blobs, ...) are being served from an external storage.')
+      %p
+        = s_('AccessTokens|It cannot be used to access any other data.')
+    .col-lg-8
+      = label_tag :static_object_token, s_('AccessTokens|Static object token'), class: "label-bold"
+      = text_field_tag :static_object_token, current_user.static_object_token, class: 'form-control', readonly: true, onclick: 'this.select()'
+      %p.form-text.text-muted
+        - reset_link = url_for [:reset, :static_object_token, :profile]
+        - reset_link_start = '<a data-confirm="%{confirm}" rel="nofollow" data-method="put" href="%{url}">'.html_safe % { confirm: s_('AccessTokens|Are you sure?'), url: reset_link }
+        - reset_link_end = '</a>'.html_safe
+        - reset_message = s_('AccessTokens|Keep this token secret. Anyone who gets ahold of it can access repository static objects as if they were you. You should %{reset_link_start}reset it%{reset_link_end} if that ever happens.') % { reset_link_start: reset_link_start, reset_link_end: reset_link_end }
+        = reset_message.html_safe
diff --git a/app/views/projects/buttons/_download_links.html.haml b/app/views/projects/buttons/_download_links.html.haml
index d344167a6c5..b256d94065b 100644
--- a/app/views/projects/buttons/_download_links.html.haml
+++ b/app/views/projects/buttons/_download_links.html.haml
@@ -2,4 +2,5 @@
 
 .btn-group.ml-0.w-100
   - formats.each do |(fmt, extra_class)|
-    = link_to fmt, project_archive_path(project, id: tree_join(ref, archive_prefix), path: path, format: fmt), rel: 'nofollow', download: '', class: "btn btn-xs #{extra_class}"
+    - archive_path = project_archive_path(project, id: tree_join(ref, archive_prefix), path: path, format: fmt)
+    = link_to fmt, external_storage_url_or_path(archive_path), rel: 'nofollow', download: '', class: "btn btn-xs #{extra_class}"