diff options
| author | Stan Hu <stanhu@gmail.com> | 2018-07-17 12:44:02 -0700 |
|---|---|---|
| committer | Stan Hu <stanhu@gmail.com> | 2018-07-17 12:44:02 -0700 |
| commit | 282904592f1e0e4f98ddeb1a829a1b900cf63451 (patch) | |
| tree | 3a48182d7b9ba8b3c1d59c8c0e06e429d329bc15 /lib | |
| parent | 11dd390c116f163a2e8a46af83e0d1bb194fb1f7 (diff) | |
| parent | 2f16eab03d8af372ae8461de5abf91b480ffd761 (diff) | |
| download | gitlab-ce-282904592f1e0e4f98ddeb1a829a1b900cf63451.tar.gz | |
Merge branch 'master' into sh-support-bitbucket-server-import
Diffstat (limited to 'lib')
26 files changed, 378 insertions, 342 deletions
diff --git a/lib/additional_email_headers_interceptor.rb b/lib/additional_email_headers_interceptor.rb deleted file mode 100644 index 3cb1694b9f1..00000000000 --- a/lib/additional_email_headers_interceptor.rb +++ /dev/null @@ -1,6 +0,0 @@ -class AdditionalEmailHeadersInterceptor - def self.delivering_email(message) - message.header['Auto-Submitted'] ||= 'auto-generated' - message.header['X-Auto-Response-Suppress'] ||= 'All' - end -end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 9c53b7c3fe7..f7737468148 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -385,7 +385,7 @@ module API finder_params[:non_public] = true if params[:membership].present? finder_params[:starred] = true if params[:starred].present? finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] - finder_params[:archived] = params[:archived] + finder_params[:archived] = archived_param unless params[:archived].nil? finder_params[:search] = params[:search] if params[:search] finder_params[:user] = params.delete(:user) if params[:user] finder_params[:custom_attributes] = params[:custom_attributes] if params[:custom_attributes] @@ -496,5 +496,11 @@ module API exception.status == 500 end + + def archived_param + return 'only' if params[:archived] + + params[:archived] + end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 8273abe48c9..0888e3befac 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -30,7 +30,7 @@ module API end params :filter_params do - optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :archived, type: Boolean, desc: 'Limit by archived status' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'Limit by visibility' optional :search, type: String, desc: 'Return list of projects matching the search criteria' diff --git a/lib/banzai/filter/blockquote_fence_filter.rb b/lib/banzai/filter/blockquote_fence_filter.rb index fbfcd72c916..7108e828c6d 100644 --- a/lib/banzai/filter/blockquote_fence_filter.rb +++ b/lib/banzai/filter/blockquote_fence_filter.rb @@ -2,27 +2,7 @@ module Banzai module Filter class BlockquoteFenceFilter < HTML::Pipeline::TextFilter REGEX = %r{ - (?<code> - # Code blocks: - # ``` - # Anything, including `>>>` blocks which are ignored by this filter - # ``` - - ^``` - .+? - \n```\ *$ - ) - | - (?<html> - # HTML block: - # <tag> - # Anything, including `>>>` blocks which are ignored by this filter - # </tag> - - ^<[^>]+?>\ *\n - .+? - \n<\/[^>]+?>\ *$ - ) + #{::Gitlab::Regex.markdown_code_or_html_blocks} | (?: # Blockquote: diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 0d9b874ef85..5dab80dd3eb 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -24,6 +24,17 @@ module Banzai Filter::AutolinkFilter, Filter::ExternalLinkFilter, + *reference_filters, + + Filter::TaskListFilter, + Filter::InlineDiffFilter, + + Filter::SetDirectionFilter + ] + end + + def self.reference_filters + [ Filter::UserReferenceFilter, Filter::IssueReferenceFilter, Filter::ExternalIssueReferenceFilter, @@ -32,12 +43,7 @@ module Banzai Filter::CommitRangeReferenceFilter, Filter::CommitReferenceFilter, Filter::LabelReferenceFilter, - Filter::MilestoneReferenceFilter, - - Filter::TaskListFilter, - Filter::InlineDiffFilter, - - Filter::SetDirectionFilter + Filter::MilestoneReferenceFilter ] end diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index dcd52bc03c7..0b2e584ef16 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -2,11 +2,17 @@ module Banzai module Pipeline class PostProcessPipeline < BasePipeline def self.filters - FilterArray[ + @filters ||= FilterArray[ + *internal_link_filters, + Filter::AbsoluteLinkFilter + ] + end + + def self.internal_link_filters + [ Filter::RedactorFilter, Filter::RelativeLinkFilter, - Filter::IssuableStateFilter, - Filter::AbsoluteLinkFilter + Filter::IssuableStateFilter ] end diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index 1929099931b..cd5a6c8875c 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -10,13 +10,19 @@ module Banzai Filter::AutolinkFilter, Filter::ExternalLinkFilter, + *reference_filters + ] + end + + def self.reference_filters + [ Filter::UserReferenceFilter, Filter::IssueReferenceFilter, Filter::ExternalIssueReferenceFilter, Filter::MergeRequestReferenceFilter, Filter::SnippetReferenceFilter, Filter::CommitRangeReferenceFilter, - Filter::CommitReferenceFilter, + Filter::CommitReferenceFilter ] end end diff --git a/lib/disable_email_interceptor.rb b/lib/disable_email_interceptor.rb deleted file mode 100644 index cee664b8951..00000000000 --- a/lib/disable_email_interceptor.rb +++ /dev/null @@ -1,7 +0,0 @@ -# Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails -class DisableEmailInterceptor - def self.delivering_email(message) - message.perform_deliveries = false - Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" - end -end diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb deleted file mode 100644 index 3978a6d9fe4..00000000000 --- a/lib/email_template_interceptor.rb +++ /dev/null @@ -1,11 +0,0 @@ -# Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails -class EmailTemplateInterceptor - def self.delivering_email(message) - # Remove HTML part if HTML emails are disabled. - unless Gitlab::CurrentSettings.html_emails_enabled - message.parts.delete_if do |part| - part.content_type.start_with?('text/html') - end - end - end -end diff --git a/lib/gitlab/email/hook/additional_headers_interceptor.rb b/lib/gitlab/email/hook/additional_headers_interceptor.rb new file mode 100644 index 00000000000..064cb5e659a --- /dev/null +++ b/lib/gitlab/email/hook/additional_headers_interceptor.rb @@ -0,0 +1,12 @@ +module Gitlab + module Email + module Hook + class AdditionalHeadersInterceptor + def self.delivering_email(message) + message.header['Auto-Submitted'] ||= 'auto-generated' + message.header['X-Auto-Response-Suppress'] ||= 'All' + end + end + end + end +end diff --git a/lib/gitlab/email/hook/delivery_metrics_observer.rb b/lib/gitlab/email/hook/delivery_metrics_observer.rb new file mode 100644 index 00000000000..1c2985f6045 --- /dev/null +++ b/lib/gitlab/email/hook/delivery_metrics_observer.rb @@ -0,0 +1,31 @@ +module Gitlab + module Email + module Hook + class DeliveryMetricsObserver + extend Gitlab::Utils::StrongMemoize + + def self.delivering_email(_message) + delivery_attempts_counter.increment + end + + def self.delivered_email(_message) + delivered_emails_counter.increment + end + + def self.delivery_attempts_counter + strong_memoize(:delivery_attempts_counter) do + Gitlab::Metrics.counter(:gitlab_emails_delivery_attempts_total, + 'Counter of total emails delivery attempts') + end + end + + def self.delivered_emails_counter + strong_memoize(:delivered_emails_counter) do + Gitlab::Metrics.counter(:gitlab_emails_delivered_total, + 'Counter of total emails delievered') + end + end + end + end + end +end diff --git a/lib/gitlab/email/hook/disable_email_interceptor.rb b/lib/gitlab/email/hook/disable_email_interceptor.rb new file mode 100644 index 00000000000..7bb8b53f0c8 --- /dev/null +++ b/lib/gitlab/email/hook/disable_email_interceptor.rb @@ -0,0 +1,13 @@ +module Gitlab + module Email + module Hook + class DisableEmailInterceptor + def self.delivering_email(message) + message.perform_deliveries = false + + Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" + end + end + end + end +end diff --git a/lib/gitlab/email/hook/email_template_interceptor.rb b/lib/gitlab/email/hook/email_template_interceptor.rb new file mode 100644 index 00000000000..be0c4dd862e --- /dev/null +++ b/lib/gitlab/email/hook/email_template_interceptor.rb @@ -0,0 +1,18 @@ +module Gitlab + module Email + module Hook + class EmailTemplateInterceptor + ## + # Remove HTML part if HTML emails are disabled. + # + def self.delivering_email(message) + unless Gitlab::CurrentSettings.html_emails_enabled + message.parts.delete_if do |part| + part.content_type.start_with?('text/html') + end + end + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index fc4711751b1..2cbd9c218d4 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -280,12 +280,6 @@ module Gitlab end.map(&:name) end - def rugged_head - rugged.head - rescue Rugged::ReferenceError - nil - end - def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:) ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) @@ -449,12 +443,8 @@ module Gitlab # Returns the SHA of the most recent common ancestor of +from+ and +to+ def merge_base(from, to) - gitaly_migrate(:merge_base) do |is_enabled| - if is_enabled - gitaly_repository_client.find_merge_base(from, to) - else - rugged_merge_base(from, to) - end + wrapped_gitaly_errors do + gitaly_repository_client.find_merge_base(from, to) end end @@ -470,12 +460,8 @@ module Gitlab return [] unless root_sha - branches = gitaly_migrate(:merged_branch_names) do |is_enabled| - if is_enabled - gitaly_merged_branch_names(branch_names, root_sha) - else - git_merged_branch_names(branch_names, root_sha) - end + branches = wrapped_gitaly_errors do + gitaly_merged_branch_names(branch_names, root_sha) end Set.new(branches) @@ -515,11 +501,6 @@ module Gitlab @refs_hash end - # Lookup for rugged object by oid or ref name - def lookup(oid_or_ref_name) - rugged.rev_parse(oid_or_ref_name) - end - # Returns url for submodule # # Ex. @@ -859,12 +840,8 @@ module Gitlab def write_ref(ref_path, ref, old_ref: nil, shell: true) ref_path = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{ref_path}" unless ref_path.start_with?("refs/") || ref_path == "HEAD" - gitaly_migrate(:write_ref) do |is_enabled| - if is_enabled - gitaly_repository_client.write_ref(ref_path, ref, old_ref, shell) - else - local_write_ref(ref_path, ref, old_ref: old_ref, shell: shell) - end + wrapped_gitaly_errors do + gitaly_repository_client.write_ref(ref_path, ref, old_ref, shell) end end @@ -916,20 +893,6 @@ module Gitlab Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit) end - def commit_index(user, branch_name, index, options) - committer = user_to_committer(user) - - OperationService.new(user, self).with_branch(branch_name) do - commit_params = options.merge( - tree: index.write_tree(rugged), - author: committer, - committer: committer - ) - - create_commit(commit_params) - end - end - def fsck msg, status = gitaly_repository_client.fsck @@ -1213,37 +1176,6 @@ module Gitlab end end - def local_write_ref(ref_path, ref, old_ref: nil, shell: true) - if shell - shell_write_ref(ref_path, ref, old_ref) - else - rugged_write_ref(ref_path, ref) - end - end - - def rugged_write_config(full_path:) - rugged.config['gitlab.fullpath'] = full_path - end - - def shell_write_ref(ref_path, ref, old_ref) - raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') - raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") - raise ArgumentError, "invalid old_ref #{old_ref.inspect}" if !old_ref.nil? && old_ref.include?("\x00") - - input = "update #{ref_path}\x00#{ref}\x00#{old_ref}\x00" - run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } - end - - def rugged_write_ref(ref_path, ref) - rugged.references.create(ref_path, ref, force: true) - rescue Rugged::ReferenceError => ex - Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" - rescue Rugged::OSError => ex - raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/ - - Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" - end - def run_git(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) cmd = [Gitlab.config.git.bin_path, *args] cmd.unshift("nice") if nice @@ -1314,20 +1246,6 @@ module Gitlab } end - def git_merged_branch_names(branch_names, root_sha) - git_arguments = - %W[branch --merged #{root_sha} - --format=%(refname:short)\ %(objectname)] + branch_names - - lines = run_git(git_arguments).first.lines - - lines.each_with_object([]) do |line, branches| - name, sha = line.strip.split(' ', 2) - - branches << name if sha != root_sha - end - end - def gitaly_merged_branch_names(branch_names, root_sha) qualified_branch_names = branch_names.map { |b| "refs/heads/#{b}" } @@ -1356,23 +1274,6 @@ module Gitlab end end - # We are trying to deprecate this method because it does a lot of work - # but it seems to be used only to look up submodule URL's. - # https://gitlab.com/gitlab-org/gitaly/issues/329 - def submodules(ref) - commit = rev_parse_target(ref) - return {} unless commit - - begin - content = blob_content(commit, ".gitmodules") - rescue InvalidBlobName - return {} - end - - parser = GitmodulesParser.new(content) - fill_submodule_ids(commit, parser.parse) - end - def gitaly_submodule_url_for(ref, path) # We don't care about the contents so 1 byte is enough. Can't request 0 bytes, 0 means unlimited. commit_object = gitaly_commit_client.tree_entry(ref, path, 1) @@ -1395,68 +1296,6 @@ module Gitlab Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact end - # Get the content of a blob for a given commit. If the blob is a commit - # (for submodules) then return the blob's OID. - def blob_content(commit, blob_name) - blob_entry = tree_entry(commit, blob_name) - - unless blob_entry - raise InvalidBlobName.new("Invalid blob name: #{blob_name}") - end - - case blob_entry[:type] - when :commit - blob_entry[:oid] - when :tree - raise InvalidBlobName.new("#{blob_name} is a tree, not a blob") - when :blob - rugged.lookup(blob_entry[:oid]).content - end - end - - # Fill in the 'id' field of a submodule hash from its values - # as-of +commit+. Return a Hash consisting only of entries - # from the submodule hash for which the 'id' field is filled. - def fill_submodule_ids(commit, submodule_data) - submodule_data.each do |path, data| - id = begin - blob_content(commit, path) - rescue InvalidBlobName - nil - end - data['id'] = id - end - submodule_data.select { |path, data| data['id'] } - end - - # Find the entry for +path+ in the tree for +commit+ - def tree_entry(commit, path) - pathname = Pathname.new(path) - first = true - tmp_entry = nil - - pathname.each_filename do |dir| - if first - tmp_entry = commit.tree[dir] - first = false - elsif tmp_entry.nil? - return nil - else - begin - tmp_entry = rugged.lookup(tmp_entry[:oid]) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - return nil - end - - return nil unless tmp_entry.type == :tree - - tmp_entry = tmp_entry[dir] - end - end - - tmp_entry - end - # Return the Rugged patches for the diff between +from+ and +to+. def diff_patches(from, to, options = {}, *paths) options ||= {} @@ -1496,75 +1335,6 @@ module Gitlab gitaly_repository_client.apply_gitattributes(revision) end - def rugged_copy_gitattributes(ref) - begin - commit = lookup(ref) - rescue Rugged::ReferenceError - raise InvalidRef.new("Ref #{ref} is invalid") - end - - # Create the paths - info_dir_path = File.join(path, 'info') - info_attributes_path = File.join(info_dir_path, 'attributes') - - begin - # Retrieve the contents of the blob - gitattributes_content = blob_content(commit, '.gitattributes') - rescue InvalidBlobName - # No .gitattributes found. Should now remove any info/attributes and return - File.delete(info_attributes_path) if File.exist?(info_attributes_path) - return - end - - # Create the info directory if needed - Dir.mkdir(info_dir_path) unless File.directory?(info_dir_path) - - # Write the contents of the .gitattributes file to info/attributes - # Use binary mode to prevent Rails from converting ASCII-8BIT to UTF-8 - File.open(info_attributes_path, "wb") do |file| - file.write(gitattributes_content) - end - end - - def rugged_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - - Gitlab::Git.check_namespace!(commit, start_repository) - - cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha) - raise CreateTreeError unless cherry_pick_tree_id - - committer = user_to_committer(user) - - create_commit(message: message, - author: { - email: commit.author_email, - name: commit.author_name, - time: commit.authored_date - }, - committer: committer, - tree: cherry_pick_tree_id, - parents: [start_commit.sha]) - end - end - - def check_cherry_pick_content(target_commit, source_sha) - args = [target_commit.sha, source_sha] - args << 1 if target_commit.merge_commit? - - cherry_pick_index = rugged.cherrypick_commit(*args) - return false if cherry_pick_index.conflicts? - - tree_id = cherry_pick_index.write_tree(rugged) - return false unless diff_exists?(source_sha, tree_id) - - tree_id - end - def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) @@ -1621,12 +1391,6 @@ module Gitlab raise CommandError, @gitlab_projects.output end - def rugged_merge_base(from, to) - rugged.merge_base(from, to) - rescue Rugged::ReferenceError - nil - end - def rev_list_param(spec) spec == :all ? ['--all'] : spec end @@ -1634,12 +1398,6 @@ module Gitlab def sha_from_ref(ref) rev_parse_target(ref).oid end - - def create_commit(params = {}) - params[:message].delete!("\r") - - Rugged::Commit.create(rugged, params) - end end end end diff --git a/lib/gitlab/hook_data/base_builder.rb b/lib/gitlab/hook_data/base_builder.rb new file mode 100644 index 00000000000..4ffca356b29 --- /dev/null +++ b/lib/gitlab/hook_data/base_builder.rb @@ -0,0 +1,38 @@ +module Gitlab + module HookData + class BaseBuilder + attr_accessor :object + + MARKDOWN_SIMPLE_IMAGE = %r{ + #{::Gitlab::Regex.markdown_code_or_html_blocks} + | + (?<image> + ! + \[(?<title>[^\n]*?)\] + \((?<url>(?!(https?://|//))[^\n]+?)\) + ) + }mx.freeze + + def initialize(object) + @object = object + end + + private + + def absolute_image_urls(markdown_text) + return markdown_text unless markdown_text.present? + + markdown_text.gsub(MARKDOWN_SIMPLE_IMAGE) do + if $~[:image] + url = $~[:url] + url = "/#{url}" unless url.start_with?('/') + + "![#{$~[:title]}](#{Gitlab.config.gitlab.url}#{url})" + else + $~[0] + end + end + end + end + end +end diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb index 6ab36676127..f2eda398b8f 100644 --- a/lib/gitlab/hook_data/issuable_builder.rb +++ b/lib/gitlab/hook_data/issuable_builder.rb @@ -1,13 +1,9 @@ module Gitlab module HookData - class IssuableBuilder + class IssuableBuilder < BaseBuilder CHANGES_KEYS = %i[previous current].freeze - attr_accessor :issuable - - def initialize(issuable) - @issuable = issuable - end + alias_method :issuable, :object def build(user: nil, changes: {}) hook_data = { diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index f9b1a3caf5e..0d71c748dc6 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -1,6 +1,6 @@ module Gitlab module HookData - class IssueBuilder + class IssueBuilder < BaseBuilder SAFE_HOOK_ATTRIBUTES = %i[ assignee_id author_id @@ -30,14 +30,11 @@ module Gitlab total_time_spent ].freeze - attr_accessor :issue - - def initialize(issue) - @issue = issue - end + alias_method :issue, :object def build attrs = { + description: absolute_image_urls(issue.description), url: Gitlab::UrlBuilder.build(issue), total_time_spent: issue.total_time_spent, human_total_time_spent: issue.human_total_time_spent, diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index aff786864f2..dfbed0597ed 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -1,6 +1,6 @@ module Gitlab module HookData - class MergeRequestBuilder + class MergeRequestBuilder < BaseBuilder SAFE_HOOK_ATTRIBUTES = %i[ assignee_id author_id @@ -35,14 +35,11 @@ module Gitlab total_time_spent ].freeze - attr_accessor :merge_request - - def initialize(merge_request) - @merge_request = merge_request - end + alias_method :merge_request, :object def build attrs = { + description: absolute_image_urls(merge_request.description), url: Gitlab::UrlBuilder.build(merge_request), source: merge_request.source_project.try(:hook_attrs), target: merge_request.target_project.hook_attrs, diff --git a/lib/gitlab/hook_data/note_builder.rb b/lib/gitlab/hook_data/note_builder.rb new file mode 100644 index 00000000000..81873e345d5 --- /dev/null +++ b/lib/gitlab/hook_data/note_builder.rb @@ -0,0 +1,43 @@ +module Gitlab + module HookData + class NoteBuilder < BaseBuilder + SAFE_HOOK_ATTRIBUTES = %i[ + attachment + author_id + change_position + commit_id + created_at + discussion_id + id + line_code + note + noteable_id + noteable_type + original_position + position + project_id + resolved_at + resolved_by_id + resolved_by_push + st_diff + system + type + updated_at + updated_by_id + ].freeze + + alias_method :note, :object + + def build + note + .attributes + .with_indifferent_access + .slice(*SAFE_HOOK_ATTRIBUTES) + .merge( + description: absolute_image_urls(note.note), + url: Gitlab::UrlBuilder.build(note) + ) + end + end + end +end diff --git a/lib/gitlab/hook_data/wiki_page_builder.rb b/lib/gitlab/hook_data/wiki_page_builder.rb new file mode 100644 index 00000000000..59c94a61cf2 --- /dev/null +++ b/lib/gitlab/hook_data/wiki_page_builder.rb @@ -0,0 +1,15 @@ +module Gitlab + module HookData + class WikiPageBuilder < BaseBuilder + alias_method :wiki_page, :object + + def build + wiki_page + .attributes + .merge( + 'content' => absolute_image_urls(wiki_page.content) + ) + end + end + end +end diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 998c21e2586..31ef0490cb3 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -11,7 +11,12 @@ module Gitlab def save return true unless @project.avatar.exists? - copy_files(avatar_path, avatar_export_path) + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared, + relative_export_path: 'avatar', + from: avatar_path + ).save rescue => e @shared.error(e) false @@ -19,10 +24,6 @@ module Gitlab private - def avatar_export_path - File.join(@shared.export_path, 'avatar', @project.avatar_identifier) - end - def avatar_path @project.avatar.path end diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb new file mode 100644 index 00000000000..1110149712d --- /dev/null +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -0,0 +1,101 @@ +module Gitlab + module ImportExport + class UploadsManager + include Gitlab::ImportExport::CommandLineUtil + + UPLOADS_BATCH_SIZE = 100 + + def initialize(project:, shared:, relative_export_path: 'uploads', from: nil) + @project = project + @shared = shared + @relative_export_path = relative_export_path + @from = from || default_uploads_path + end + + def save + copy_files(@from, uploads_export_path) if File.directory?(@from) + + if File.file?(@from) && @relative_export_path == 'avatar' + copy_files(@from, File.join(uploads_export_path, @project.avatar.filename)) + end + + copy_from_object_storage + + true + rescue => e + @shared.error(e) + false + end + + def restore + Dir["#{uploads_export_path}/**/*"].each do |upload| + next if File.directory?(upload) + + add_upload(upload) + end + + true + rescue => e + @shared.error(e) + false + end + + private + + def add_upload(upload) + uploader_context = FileUploader.extract_dynamic_path(upload).named_captures.symbolize_keys + + UploadService.new(@project, File.open(upload, 'r'), FileUploader, uploader_context).execute + end + + def copy_from_object_storage + return unless Gitlab::ImportExport.object_storage? + + each_uploader do |uploader| + next unless uploader.file + next if uploader.upload.local? # Already copied, using the old method + + download_and_copy(uploader) + end + end + + def default_uploads_path + FileUploader.absolute_base_dir(@project) + end + + def uploads_export_path + @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path) + end + + def each_uploader + avatar_path = @project.avatar&.upload&.path + + if @relative_export_path == 'avatar' + yield(@project.avatar) + else + project_uploads_except_avatar(avatar_path).find_each(batch_size: UPLOADS_BATCH_SIZE) do |upload| + yield(upload.build_uploader) + end + end + end + + def project_uploads_except_avatar(avatar_path) + return @project.uploads unless avatar_path + + @project.uploads.where("path != ?", avatar_path) + end + + def download_and_copy(upload) + secret = upload.try(:secret) || '' + upload_path = File.join(uploads_export_path, secret, upload.filename) + + mkdir_p(File.join(uploads_export_path, secret)) + + File.open(upload_path, 'w') do |file| + # Download (stream) file from the uploader's location + IO.copy_stream(URI.parse(upload.file.url).open, file) + end + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index df19354b76e..25f85936227 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -2,13 +2,30 @@ module Gitlab module ImportExport class UploadsRestorer < UploadsSaver def restore - return true unless File.directory?(uploads_export_path) + if Gitlab::ImportExport.object_storage? + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).restore + elsif File.directory?(uploads_export_path) + copy_files(uploads_export_path, uploads_path) - copy_files(uploads_export_path, uploads_path) + true + else + true # Proceed without uploads + end rescue => e @shared.error(e) false end + + def uploads_path + FileUploader.absolute_base_dir(@project) + end + + def uploads_export_path + @uploads_export_path ||= File.join(@shared.export_path, 'uploads') + end end end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 2f08dda55fd..b3f17af5661 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -9,21 +9,14 @@ module Gitlab end def save - return true unless File.directory?(uploads_path) - - copy_files(uploads_path, uploads_export_path) + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).save rescue => e @shared.error(e) false end - - def uploads_path - FileUploader.absolute_base_dir(@project) - end - - def uploads_export_path - File.join(@shared.export_path, 'uploads') - end end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 38f119cf06d..c205f348023 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -20,7 +20,7 @@ module Gitlab define_histogram :gitlab_sql_duration_seconds do docstring 'SQL time' base_labels Transaction::BASE_LABELS - buckets [0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0] + buckets [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0] end def current_transaction diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ac3de2a8f71..e1a958c508a 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -73,5 +73,31 @@ module Gitlab def build_trace_section_regex @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)\r\033\[0K/.freeze end + + def markdown_code_or_html_blocks + @markdown_code_or_html_blocks ||= %r{ + (?<code> + # Code blocks: + # ``` + # Anything, including `>>>` blocks which are ignored by this filter + # ``` + + ^``` + .+? + \n```\ *$ + ) + | + (?<html> + # HTML block: + # <tag> + # Anything, including `>>>` blocks which are ignored by this filter + # </tag> + + ^<[^>]+?>\ *\n + .+? + \n<\/[^>]+?>\ *$ + ) + }mx + end end end |