summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorBrett Walker <bwalker@gitlab.com>2019-02-20 17:51:55 -0600
committerBrett Walker <bwalker@gitlab.com>2019-08-20 10:28:15 -0500
commitfd9c5643eb37f3f2226a28303ca8bc164030aba3 (patch)
treebdebf99b85480647e4117672e853ddc1d8d32ee8 /lib
parent66abd26046c93280cc56ebaf273e132cd919e03c (diff)
downloadgitlab-ce-fd9c5643eb37f3f2226a28303ca8bc164030aba3.tar.gz
Add support for using a Camo proxy server
User images and videos will get proxied through the Camo server in order to keep malicious sites from collecting the IP address of users.
Diffstat (limited to 'lib')
-rw-r--r--lib/api/entities.rb3
-rw-r--r--lib/api/settings.rb6
-rw-r--r--lib/api/validations/types/comma_separated_to_array.rb22
-rw-r--r--lib/banzai/filter/asset_proxy_filter.rb62
-rw-r--r--lib/banzai/filter/external_link_filter.rb18
-rw-r--r--lib/banzai/filter/image_link_filter.rb3
-rw-r--r--lib/banzai/filter/video_link_filter.rb15
-rw-r--r--lib/banzai/pipeline/ascii_doc_pipeline.rb5
-rw-r--r--lib/banzai/pipeline/gfm_pipeline.rb3
-rw-r--r--lib/banzai/pipeline/markup_pipeline.rb5
-rw-r--r--lib/banzai/pipeline/single_line_pipeline.rb3
11 files changed, 139 insertions, 6 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 09253ab6b0e..4dc7830f583 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -1171,6 +1171,9 @@ module API
attributes.delete(:performance_bar_enabled)
attributes.delete(:allow_local_requests_from_hooks_and_services)
+ # let's not expose the secret key in a response
+ attributes.delete(:asset_proxy_secret_key)
+
attributes
end
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index c36ee5af63f..b67c18b5465 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -36,6 +36,10 @@ module API
given akismet_enabled: ->(val) { val } do
requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com'
end
+ optional :asset_proxy_enabled, type: Boolean, desc: 'Enable proxying of assets'
+ optional :asset_proxy_url, type: String, desc: 'URL of the asset proxy server'
+ optional :asset_proxy_secret_key, type: String, desc: 'Shared secret with the asset proxy server'
+ optional :asset_proxy_whitelist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically whitelisted.'
optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)'
optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts"
optional :default_project_creation, type: Integer, values: ::Gitlab::Access.project_creation_values, desc: 'Determine if developers can create projects in the group'
@@ -123,7 +127,7 @@ module API
optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.'
optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.'
optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins'
- optional :local_markdown_version, type: Integer, desc: "Local markdown version, increase this value when any cached markdown should be invalidated"
+ optional :local_markdown_version, type: Integer, desc: 'Local markdown version, increase this value when any cached markdown should be invalidated'
optional :allow_local_requests_from_hooks_and_services, type: Boolean, desc: 'Deprecated: Use :allow_local_requests_from_web_hooks_and_services instead. Allow requests to the local network from hooks and services.' # support legacy names, can be removed in v5
optional :snowplow_enabled, type: Grape::API::Boolean, desc: 'Enable Snowplow tracking'
given snowplow_enabled: ->(val) { val } do
diff --git a/lib/api/validations/types/comma_separated_to_array.rb b/lib/api/validations/types/comma_separated_to_array.rb
new file mode 100644
index 00000000000..b551878abd1
--- /dev/null
+++ b/lib/api/validations/types/comma_separated_to_array.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module API
+ module Validations
+ module Types
+ class CommaSeparatedToArray
+ def self.coerce
+ lambda do |value|
+ case value
+ when String
+ value.split(',').map(&:strip)
+ when Array
+ value.map { |v| v.to_s.split(',').map(&:strip) }.flatten
+ else
+ []
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/asset_proxy_filter.rb b/lib/banzai/filter/asset_proxy_filter.rb
new file mode 100644
index 00000000000..0a9a52a73a1
--- /dev/null
+++ b/lib/banzai/filter/asset_proxy_filter.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ # Proxy's images/assets to another server. Reduces mixed content warnings
+ # as well as hiding the customer's IP address when requesting images.
+ # Copies the original img `src` to `data-canonical-src` then replaces the
+ # `src` with a new url to the proxy server.
+ class AssetProxyFilter < HTML::Pipeline::CamoFilter
+ def initialize(text, context = nil, result = nil)
+ super
+ end
+
+ def validate
+ needs(:asset_proxy, :asset_proxy_secret_key) if asset_proxy_enabled?
+ end
+
+ def asset_host_whitelisted?(host)
+ context[:asset_proxy_domain_regexp] ? context[:asset_proxy_domain_regexp].match?(host) : false
+ end
+
+ def self.transform_context(context)
+ context[:disable_asset_proxy] = !Gitlab.config.asset_proxy.enabled
+
+ unless context[:disable_asset_proxy]
+ context[:asset_proxy_enabled] = !context[:disable_asset_proxy]
+ context[:asset_proxy] = Gitlab.config.asset_proxy.url
+ context[:asset_proxy_secret_key] = Gitlab.config.asset_proxy.secret_key
+ context[:asset_proxy_domain_regexp] = Gitlab.config.asset_proxy.domain_regexp
+ end
+
+ context
+ end
+
+ # called during an initializer. Caching the values in Gitlab.config significantly increased
+ # performance, rather than querying Gitlab::CurrentSettings.current_application_settings
+ # over and over. However, this does mean that the Rails servers need to get restarted
+ # whenever the application settings are changed
+ def self.initialize_settings
+ application_settings = Gitlab::CurrentSettings.current_application_settings
+ Gitlab.config['asset_proxy'] ||= Settingslogic.new({})
+
+ if application_settings.respond_to?(:asset_proxy_enabled)
+ Gitlab.config.asset_proxy['enabled'] = application_settings.asset_proxy_enabled
+ Gitlab.config.asset_proxy['url'] = application_settings.asset_proxy_url
+ Gitlab.config.asset_proxy['secret_key'] = application_settings.asset_proxy_secret_key
+ Gitlab.config.asset_proxy['whitelist'] = application_settings.asset_proxy_whitelist || [Gitlab.config.gitlab.host]
+ Gitlab.config.asset_proxy['domain_regexp'] = compile_whitelist(Gitlab.config.asset_proxy.whitelist)
+ else
+ Gitlab.config.asset_proxy['enabled'] = ::ApplicationSetting.defaults[:asset_proxy_enabled]
+ end
+ end
+
+ def self.compile_whitelist(domain_list)
+ return if domain_list.empty?
+
+ escaped = domain_list.map { |domain| Regexp.escape(domain).gsub('\*', '.*?') }
+ Regexp.new("^(#{escaped.join('|')})$", Regexp::IGNORECASE)
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb
index 61ee3eac216..fb721fe12b1 100644
--- a/lib/banzai/filter/external_link_filter.rb
+++ b/lib/banzai/filter/external_link_filter.rb
@@ -14,10 +14,10 @@ module Banzai
# such as on `mailto:` links. Since we've been using it, do an
# initial parse for validity and then use Addressable
# for IDN support, etc
- uri = uri_strict(node['href'].to_s)
+ uri = uri_strict(node_src(node))
if uri
- node.set_attribute('href', uri.to_s)
- addressable_uri = addressable_uri(node['href'])
+ node.set_attribute(node_src_attribute(node), uri.to_s)
+ addressable_uri = addressable_uri(node_src(node))
else
addressable_uri = nil
end
@@ -35,6 +35,16 @@ module Banzai
private
+ # if this is a link to a proxied image, then `src` is already the correct
+ # proxied url, so work with the `data-canonical-src`
+ def node_src_attribute(node)
+ node['data-canonical-src'] ? 'data-canonical-src' : 'href'
+ end
+
+ def node_src(node)
+ node[node_src_attribute(node)]
+ end
+
def uri_strict(href)
URI.parse(href)
rescue URI::Error
@@ -72,7 +82,7 @@ module Banzai
return unless uri
return unless context[:emailable_links]
- unencoded_uri_str = Addressable::URI.unencode(node['href'])
+ unencoded_uri_str = Addressable::URI.unencode(node_src(node))
if unencoded_uri_str == node.content && idn?(uri)
node.content = uri.normalize
diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb
index 01237303c27..ed0a01e6277 100644
--- a/lib/banzai/filter/image_link_filter.rb
+++ b/lib/banzai/filter/image_link_filter.rb
@@ -18,6 +18,9 @@ module Banzai
rel: 'noopener noreferrer'
)
+ # make sure the original non-proxied src carries over to the link
+ link['data-canonical-src'] = img['data-canonical-src'] if img['data-canonical-src']
+
link.children = img.clone
img.replace(link)
diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb
index 0fff104cf91..a278fcfdb47 100644
--- a/lib/banzai/filter/video_link_filter.rb
+++ b/lib/banzai/filter/video_link_filter.rb
@@ -23,6 +23,14 @@ module Banzai
"'.#{ext}' = substring(@src, string-length(@src) - #{ext.size})"
end
+ if context[:asset_proxy_enabled].present?
+ src_query.concat(
+ UploaderHelper::VIDEO_EXT.map do |ext|
+ "'.#{ext}' = substring(@data-canonical-src, string-length(@data-canonical-src) - #{ext.size})"
+ end
+ )
+ end
+
"descendant-or-self::img[not(ancestor::a) and (#{src_query.join(' or ')})]"
end
end
@@ -48,6 +56,13 @@ module Banzai
target: '_blank',
rel: 'noopener noreferrer',
title: "Download '#{element['title'] || element['alt']}'")
+
+ # make sure the original non-proxied src carries over
+ if element['data-canonical-src']
+ video['data-canonical-src'] = element['data-canonical-src']
+ link['data-canonical-src'] = element['data-canonical-src']
+ end
+
download_paragraph = doc.document.create_element('p')
download_paragraph.children = link
diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb
index d25b74b23b2..82b99d3de4a 100644
--- a/lib/banzai/pipeline/ascii_doc_pipeline.rb
+++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb
@@ -6,12 +6,17 @@ module Banzai
def self.filters
FilterArray[
Filter::AsciiDocSanitizationFilter,
+ Filter::AssetProxyFilter,
Filter::SyntaxHighlightFilter,
Filter::ExternalLinkFilter,
Filter::PlantumlFilter,
Filter::AsciiDocPostProcessingFilter
]
end
+
+ def self.transform_context(context)
+ Filter::AssetProxyFilter.transform_context(context)
+ end
end
end
end
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index 2c1006f708a..f419e54c264 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -17,6 +17,7 @@ module Banzai
Filter::SpacedLinkFilter,
Filter::SanitizationFilter,
+ Filter::AssetProxyFilter,
Filter::SyntaxHighlightFilter,
Filter::MathFilter,
@@ -60,7 +61,7 @@ module Banzai
def self.transform_context(context)
context[:only_path] = true unless context.key?(:only_path)
- context
+ Filter::AssetProxyFilter.transform_context(context)
end
end
end
diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb
index ceba082cd4f..c86d5f08ded 100644
--- a/lib/banzai/pipeline/markup_pipeline.rb
+++ b/lib/banzai/pipeline/markup_pipeline.rb
@@ -6,11 +6,16 @@ module Banzai
def self.filters
@filters ||= FilterArray[
Filter::SanitizationFilter,
+ Filter::AssetProxyFilter,
Filter::ExternalLinkFilter,
Filter::PlantumlFilter,
Filter::SyntaxHighlightFilter
]
end
+
+ def self.transform_context(context)
+ Filter::AssetProxyFilter.transform_context(context)
+ end
end
end
end
diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb
index 72374207a8f..9aff6880f56 100644
--- a/lib/banzai/pipeline/single_line_pipeline.rb
+++ b/lib/banzai/pipeline/single_line_pipeline.rb
@@ -7,6 +7,7 @@ module Banzai
@filters ||= FilterArray[
Filter::HtmlEntityFilter,
Filter::SanitizationFilter,
+ Filter::AssetProxyFilter,
Filter::EmojiFilter,
Filter::AutolinkFilter,
@@ -29,6 +30,8 @@ module Banzai
end
def self.transform_context(context)
+ context = Filter::AssetProxyFilter.transform_context(context)
+
super(context).merge(
no_sourcepos: true
)