From 0b12a5312c9701fbfed25fbb334d47900ced736b Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 14 Jan 2020 21:07:45 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- lib/api/helpers/members_helpers.rb | 4 + lib/api/members.rb | 4 +- lib/banzai/filter/base_relative_link_filter.rb | 45 ++++ lib/banzai/filter/relative_link_filter.rb | 277 --------------------- lib/banzai/filter/repository_link_filter.rb | 208 ++++++++++++++++ lib/banzai/filter/upload_link_filter.rb | 61 +++++ lib/banzai/pipeline/post_process_pipeline.rb | 5 +- lib/banzai/pipeline/relative_link_pipeline.rb | 13 - .../migrate_fingerprint_sha256_within_keys.rb | 16 +- 9 files changed, 333 insertions(+), 300 deletions(-) create mode 100644 lib/banzai/filter/base_relative_link_filter.rb delete mode 100644 lib/banzai/filter/relative_link_filter.rb create mode 100644 lib/banzai/filter/repository_link_filter.rb create mode 100644 lib/banzai/filter/upload_link_filter.rb delete mode 100644 lib/banzai/pipeline/relative_link_pipeline.rb (limited to 'lib') diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 3ccae8d85cc..d06c59907b4 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -41,6 +41,10 @@ module API GroupMembersFinder.new(group).execute end + def create_member(current_user, user, source, params) + source.add_user(user, params[:access_level], current_user: current_user, expires_at: params[:expires_at]) + end + def present_members(members) present members, with: Entities::Member, current_user: current_user end diff --git a/lib/api/members.rb b/lib/api/members.rb index e745bd0d4a9..e4df2f341c6 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -101,12 +101,12 @@ module API user = User.find_by_id(params[:user_id]) not_found!('User') unless user - member = source.add_user(user, params[:access_level], current_user: current_user, expires_at: params[:expires_at]) + member = create_member(current_user, user, source, params) if !member not_allowed! # This currently can only be reached in EE elsif member.persisted? && member.valid? - present_members member + present_members(member) else render_validation_error!(member) end diff --git a/lib/banzai/filter/base_relative_link_filter.rb b/lib/banzai/filter/base_relative_link_filter.rb new file mode 100644 index 00000000000..eca105ce9d9 --- /dev/null +++ b/lib/banzai/filter/base_relative_link_filter.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'uri' + +module Banzai + module Filter + class BaseRelativeLinkFilter < HTML::Pipeline::Filter + include Gitlab::Utils::StrongMemoize + + protected + + def linkable_attributes + strong_memoize(:linkable_attributes) do + attrs = [] + + attrs += doc.search('a:not(.gfm)').map do |el| + el.attribute('href') + end + + attrs += doc.search('img:not(.gfm), video:not(.gfm), audio:not(.gfm)').flat_map do |el| + [el.attribute('src'), el.attribute('data-src')] + end + + attrs.reject do |attr| + attr.blank? || attr.value.start_with?('//') + end + end + end + + def relative_url_root + Gitlab.config.gitlab.relative_url_root.presence || '/' + end + + def project + context[:project] + end + + private + + def unescape_and_scrub_uri(uri) + Addressable::URI.unescape(uri).scrub + end + end + end +end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb deleted file mode 100644 index 4f257189f8e..00000000000 --- a/lib/banzai/filter/relative_link_filter.rb +++ /dev/null @@ -1,277 +0,0 @@ -# frozen_string_literal: true - -require 'uri' - -module Banzai - module Filter - # HTML filter that "fixes" relative links to uploads or files in a repository. - # - # Context options: - # :commit - # :group - # :current_user - # :project - # :project_wiki - # :ref - # :requested_path - class RelativeLinkFilter < HTML::Pipeline::Filter - include Gitlab::Utils::StrongMemoize - - def call - return doc if context[:system_note] - - clear_memoization(:linkable_files) - clear_memoization(:linkable_attributes) - - load_uri_types - - linkable_attributes.each do |attr| - process_link_attr(attr) - end - - doc - end - - protected - - def load_uri_types - return unless linkable_files? - return unless linkable_attributes.present? - return {} unless repository - - @uri_types = request_path.present? ? get_uri_types([request_path]) : {} - - paths = linkable_attributes.flat_map do |attr| - [get_uri(attr).to_s, relative_file_path(get_uri(attr))] - end - - paths.reject!(&:blank?) - paths.uniq! - - @uri_types.merge!(get_uri_types(paths)) - end - - def linkable_files? - strong_memoize(:linkable_files) do - context[:project_wiki].nil? && repository.try(:exists?) && !repository.empty? - end - end - - def linkable_attributes - strong_memoize(:linkable_attributes) do - attrs = [] - - attrs += doc.search('a:not(.gfm)').map do |el| - el.attribute('href') - end - - attrs += doc.search('img, video, audio').flat_map do |el| - [el.attribute('src'), el.attribute('data-src')] - end - - attrs.reject do |attr| - attr.blank? || attr.value.start_with?('//') - end - end - end - - def get_uri_types(paths) - return {} if paths.empty? - - uri_types = Hash[paths.collect { |name| [name, nil] }] - - get_blob_types(paths).each do |name, type| - if type == :blob - blob = ::Blob.decorate(Gitlab::Git::Blob.new(name: name), project) - uri_types[name] = blob.image? || blob.video? || blob.audio? ? :raw : :blob - else - uri_types[name] = type - end - end - - uri_types - end - - def get_blob_types(paths) - revision_paths = paths.collect do |path| - [current_commit.sha, path.chomp("/")] - end - - Gitlab::GitalyClient::BlobService.new(repository).get_blob_types(revision_paths, 1) - end - - def get_uri(html_attr) - uri = URI(html_attr.value) - - uri if uri.relative? && uri.path.present? - rescue URI::Error, Addressable::URI::InvalidURIError - end - - def process_link_attr(html_attr) - if html_attr.value.start_with?('/uploads/') - process_link_to_upload_attr(html_attr) - elsif linkable_files? && repo_visible_to_user? - process_link_to_repository_attr(html_attr) - end - end - - def process_link_to_upload_attr(html_attr) - path_parts = [unescape_and_scrub_uri(html_attr.value)] - - if project - path_parts.unshift(relative_url_root, project.full_path) - elsif group - path_parts.unshift(relative_url_root, 'groups', group.full_path, '-') - else - path_parts.unshift(relative_url_root) - end - - begin - path = Addressable::URI.escape(File.join(*path_parts)) - rescue Addressable::URI::InvalidURIError - return - end - - html_attr.value = - if context[:only_path] - path - else - Addressable::URI.join(Gitlab.config.gitlab.base_url, path).to_s - end - end - - def process_link_to_repository_attr(html_attr) - uri = URI(html_attr.value) - - if uri.relative? && uri.path.present? - html_attr.value = rebuild_relative_uri(uri).to_s - end - rescue URI::Error, Addressable::URI::InvalidURIError - # noop - end - - def rebuild_relative_uri(uri) - file_path = nested_file_path_if_exists(uri) - - uri.path = [ - relative_url_root, - project.full_path, - uri_type(file_path), - Addressable::URI.escape(ref).gsub('#', '%23'), - Addressable::URI.escape(file_path) - ].compact.join('/').squeeze('/').chomp('/') - - uri - end - - def nested_file_path_if_exists(uri) - path = cleaned_file_path(uri) - nested_path = relative_file_path(uri) - - file_exists?(nested_path) ? nested_path : path - end - - def cleaned_file_path(uri) - unescape_and_scrub_uri(uri.path).delete("\0").chomp("/") - end - - def relative_file_path(uri) - return if uri.nil? - - build_relative_path(cleaned_file_path(uri), request_path) - end - - def request_path - return unless context[:requested_path] - - unescape_and_scrub_uri(context[:requested_path]).chomp("/") - end - - # Convert a relative path into its correct location based on the currently - # requested path - # - # path - Relative path String - # request_path - Currently-requested path String - # - # Examples: - # - # # File in the same directory as the current path - # build_relative_path("users.md", "doc/api/README.md") - # # => "doc/api/users.md" - # - # # File in the same directory, which is also the current path - # build_relative_path("users.md", "doc/api") - # # => "doc/api/users.md" - # - # # Going up one level to a different directory - # build_relative_path("../update/7.14-to-8.0.md", "doc/api/README.md") - # # => "doc/update/7.14-to-8.0.md" - # - # Returns a String - def build_relative_path(path, request_path) - return request_path if path.empty? - return path unless request_path - return path[1..-1] if path.start_with?('/') - - parts = request_path.split('/') - - parts.pop if uri_type(request_path) != :tree - - path.sub!(%r{\A\./}, '') - - while path.start_with?('../') - parts.pop - path.sub!('../', '') - end - - parts.push(path).join('/') - end - - def file_exists?(path) - path.present? && uri_type(path).present? - end - - def uri_type(path) - @uri_types[path] == :unknown ? "" : @uri_types[path] - end - - def current_commit - @current_commit ||= context[:commit] || repository.commit(ref) - end - - def relative_url_root - Gitlab.config.gitlab.relative_url_root.presence || '/' - end - - def repo_visible_to_user? - project && Ability.allowed?(current_user, :download_code, project) - end - - def ref - context[:ref] || project.default_branch - end - - def group - context[:group] - end - - def project - context[:project] - end - - def current_user - context[:current_user] - end - - def repository - @repository ||= project&.repository - end - - private - - def unescape_and_scrub_uri(uri) - Addressable::URI.unescape(uri).scrub - end - end - end -end diff --git a/lib/banzai/filter/repository_link_filter.rb b/lib/banzai/filter/repository_link_filter.rb new file mode 100644 index 00000000000..14cd607cc50 --- /dev/null +++ b/lib/banzai/filter/repository_link_filter.rb @@ -0,0 +1,208 @@ +# frozen_string_literal: true + +require 'uri' + +module Banzai + module Filter + # HTML filter that "fixes" relative links to files in a repository. + # + # Context options: + # :commit + # :current_user + # :project + # :project_wiki + # :ref + # :requested_path + # :system_note + class RepositoryLinkFilter < BaseRelativeLinkFilter + def call + return doc if context[:system_note] + + clear_memoization(:linkable_files) + clear_memoization(:linkable_attributes) + + load_uri_types + + linkable_attributes.each do |attr| + if linkable_files? && repo_visible_to_user? + process_link_to_repository_attr(attr) + end + end + + doc + end + + protected + + def load_uri_types + return unless linkable_attributes.present? + return unless linkable_files? + return {} unless repository + + @uri_types = request_path.present? ? get_uri_types([request_path]) : {} + + paths = linkable_attributes.flat_map do |attr| + [get_uri(attr).to_s, relative_file_path(get_uri(attr))] + end + + paths.reject!(&:blank?) + paths.uniq! + + @uri_types.merge!(get_uri_types(paths)) + end + + def linkable_files? + strong_memoize(:linkable_files) do + context[:project_wiki].nil? && repository.try(:exists?) && !repository.empty? + end + end + + def get_uri_types(paths) + return {} if paths.empty? + + uri_types = Hash[paths.collect { |name| [name, nil] }] + + get_blob_types(paths).each do |name, type| + if type == :blob + blob = ::Blob.decorate(Gitlab::Git::Blob.new(name: name), project) + uri_types[name] = blob.image? || blob.video? || blob.audio? ? :raw : :blob + else + uri_types[name] = type + end + end + + uri_types + end + + def get_blob_types(paths) + revision_paths = paths.collect do |path| + [current_commit.sha, path.chomp("/")] + end + + Gitlab::GitalyClient::BlobService.new(repository).get_blob_types(revision_paths, 1) + end + + def get_uri(html_attr) + uri = URI(html_attr.value) + + uri if uri.relative? && uri.path.present? + rescue URI::Error, Addressable::URI::InvalidURIError + end + + def process_link_to_repository_attr(html_attr) + uri = URI(html_attr.value) + + if uri.relative? && uri.path.present? + html_attr.value = rebuild_relative_uri(uri).to_s + end + rescue URI::Error, Addressable::URI::InvalidURIError + # noop + end + + def rebuild_relative_uri(uri) + file_path = nested_file_path_if_exists(uri) + + uri.path = [ + relative_url_root, + project.full_path, + uri_type(file_path), + Addressable::URI.escape(ref).gsub('#', '%23'), + Addressable::URI.escape(file_path) + ].compact.join('/').squeeze('/').chomp('/') + + uri + end + + def nested_file_path_if_exists(uri) + path = cleaned_file_path(uri) + nested_path = relative_file_path(uri) + + file_exists?(nested_path) ? nested_path : path + end + + def cleaned_file_path(uri) + unescape_and_scrub_uri(uri.path).delete("\0").chomp("/") + end + + def relative_file_path(uri) + return if uri.nil? + + build_relative_path(cleaned_file_path(uri), request_path) + end + + def request_path + return unless context[:requested_path] + + unescape_and_scrub_uri(context[:requested_path]).chomp("/") + end + + # Convert a relative path into its correct location based on the currently + # requested path + # + # path - Relative path String + # request_path - Currently-requested path String + # + # Examples: + # + # # File in the same directory as the current path + # build_relative_path("users.md", "doc/api/README.md") + # # => "doc/api/users.md" + # + # # File in the same directory, which is also the current path + # build_relative_path("users.md", "doc/api") + # # => "doc/api/users.md" + # + # # Going up one level to a different directory + # build_relative_path("../update/7.14-to-8.0.md", "doc/api/README.md") + # # => "doc/update/7.14-to-8.0.md" + # + # Returns a String + def build_relative_path(path, request_path) + return request_path if path.empty? + return path unless request_path + return path[1..-1] if path.start_with?('/') + + parts = request_path.split('/') + + parts.pop if uri_type(request_path) != :tree + + path.sub!(%r{\A\./}, '') + + while path.start_with?('../') + parts.pop + path.sub!('../', '') + end + + parts.push(path).join('/') + end + + def file_exists?(path) + path.present? && uri_type(path).present? + end + + def uri_type(path) + @uri_types[path] == :unknown ? "" : @uri_types[path] + end + + def current_commit + @current_commit ||= context[:commit] || repository.commit(ref) + end + + def repo_visible_to_user? + project && Ability.allowed?(current_user, :download_code, project) + end + + def ref + context[:ref] || project.default_branch + end + + def current_user + context[:current_user] + end + + def repository + @repository ||= project&.repository + end + end + end +end diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb new file mode 100644 index 00000000000..023c1288367 --- /dev/null +++ b/lib/banzai/filter/upload_link_filter.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'uri' + +module Banzai + module Filter + # HTML filter that "fixes" links to uploads. + # + # Context options: + # :group + # :only_path + # :project + # :system_note + class UploadLinkFilter < BaseRelativeLinkFilter + def call + return doc if context[:system_note] + + linkable_attributes.each do |attr| + process_link_to_upload_attr(attr) + end + + doc + end + + protected + + def process_link_to_upload_attr(html_attr) + return unless html_attr.value.start_with?('/uploads/') + + path_parts = [unescape_and_scrub_uri(html_attr.value)] + + if project + path_parts.unshift(relative_url_root, project.full_path) + elsif group + path_parts.unshift(relative_url_root, 'groups', group.full_path, '-') + else + path_parts.unshift(relative_url_root) + end + + begin + path = Addressable::URI.escape(File.join(*path_parts)) + rescue Addressable::URI::InvalidURIError + return + end + + html_attr.value = + if context[:only_path] + path + else + Addressable::URI.join(Gitlab.config.gitlab.base_url, path).to_s + end + + html_attr.parent.add_class('gfm') + end + + def group + context[:group] + end + end + end +end diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index fe629a23ff1..5e02d972614 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -16,7 +16,10 @@ module Banzai [ Filter::ReferenceRedactorFilter, Filter::InlineMetricsRedactorFilter, - Filter::RelativeLinkFilter, + # UploadLinkFilter must come before RepositoryLinkFilter to + # prevent unnecessary Gitaly calls from being made. + Filter::UploadLinkFilter, + Filter::RepositoryLinkFilter, Filter::IssuableStateFilter, Filter::SuggestionFilter ] diff --git a/lib/banzai/pipeline/relative_link_pipeline.rb b/lib/banzai/pipeline/relative_link_pipeline.rb deleted file mode 100644 index 88651892acc..00000000000 --- a/lib/banzai/pipeline/relative_link_pipeline.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Pipeline - class RelativeLinkPipeline < BasePipeline - def self.filters - FilterArray[ - Filter::RelativeLinkFilter - ] - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb index 40c109207a9..899f381e911 100644 --- a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb +++ b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb @@ -24,12 +24,14 @@ module Gitlab fingerprints = [] Key.where(id: start_id..stop_id, fingerprint_sha256: nil).find_each do |regular_key| - fingerprint = Base64.decode64(generate_ssh_public_key(regular_key.key)) - - fingerprints << { - id: regular_key.id, - fingerprint_sha256: ActiveRecord::Base.connection.escape_bytea(fingerprint) - } + if fingerprint = generate_ssh_public_key(regular_key.key) + bytea = ActiveRecord::Base.connection.escape_bytea(Base64.decode64(fingerprint)) + + fingerprints << { + id: regular_key.id, + fingerprint_sha256: bytea + } + end end Gitlab::Database.bulk_insert(TEMP_TABLE, fingerprints) @@ -48,7 +50,7 @@ module Gitlab private def generate_ssh_public_key(regular_key) - Gitlab::SSHPublicKey.new(regular_key).fingerprint("SHA256").gsub("SHA256:", "") + Gitlab::SSHPublicKey.new(regular_key).fingerprint("SHA256")&.gsub("SHA256:", "") end def execute(query) -- cgit v1.2.1