diff options
author | Paco Guzman <pacoguzmanp@gmail.com> | 2016-09-21 14:46:10 +0200 |
---|---|---|
committer | Paco Guzman <pacoguzmanp@gmail.com> | 2016-10-06 16:51:55 +0200 |
commit | b4819a5d750ee458a424cf7965926c5758bf784a (patch) | |
tree | 427a3df9f9ac740622b2000ca7e36c6f07792b21 | |
parent | 0bbeff3d5e6c1b5ea3b364f052ed6f777c3aa645 (diff) | |
download | gitlab-ce-18663-commits-reference-mentionables.tar.gz |
GitPushService group but author cross_reference creation18663-commits-reference-mentionables
The reference extractor phase happens once per type not once
pero pushed commit, so we could be avoiding a lot of DB queries
-rw-r--r-- | CHANGELOG | 1 | ||||
-rw-r--r-- | app/models/commit.rb | 4 | ||||
-rw-r--r-- | app/models/concerns/mentionable.rb | 12 | ||||
-rw-r--r-- | app/services/git_push_service.rb | 61 | ||||
-rw-r--r-- | lib/banzai/mentionable_renderer.rb | 9 | ||||
-rw-r--r-- | lib/banzai/note_renderer.rb | 3 | ||||
-rw-r--r-- | lib/banzai/object_renderer.rb | 42 | ||||
-rw-r--r-- | lib/banzai/redactor.rb | 14 | ||||
-rw-r--r-- | lib/banzai/reference_parser/base_parser.rb | 10 | ||||
-rw-r--r-- | lib/banzai/reference_querying.rb | 37 | ||||
-rw-r--r-- | lib/gitlab/cross_reference_extractor.rb | 63 | ||||
-rw-r--r-- | spec/lib/banzai/note_renderer_spec.rb | 3 | ||||
-rw-r--r-- | spec/lib/banzai/object_renderer_spec.rb | 35 | ||||
-rw-r--r-- | spec/lib/banzai/redactor_spec.rb | 4 | ||||
-rw-r--r-- | spec/lib/gitlab/cross_reference_extractor_spec.rb | 54 | ||||
-rw-r--r-- | spec/models/commit_spec.rb | 2 | ||||
-rw-r--r-- | spec/models/concerns/mentionable_spec.rb | 20 | ||||
-rw-r--r-- | spec/support/mentionable_shared_examples.rb | 2 |
18 files changed, 306 insertions, 70 deletions
diff --git a/CHANGELOG b/CHANGELOG index 68962f20d0b..9867403ddd3 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ Please view this file on the master branch, on stable branches it's out of date. v 8.13.0 (unreleased) - Update runner version only when updating contacted_at + - PostReceive worker redact cross references for commits in batch - Add link from system note to compare with previous version - Use gitlab-shell v3.6.2 (GIT TRACE logging) - Fix centering of custom header logos (Ashley Dumaine) diff --git a/app/models/commit.rb b/app/models/commit.rb index e64fd1e0c1b..41a49064276 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -171,6 +171,8 @@ class Commit end def author + return @author if defined?(@author) + if RequestStore.active? key = "commit_author:#{author_email.downcase}" # nil is a valid value since no author may exist in the system @@ -181,7 +183,7 @@ class Commit RequestStore.store[key] = @author end else - @author ||= find_author_by_any_email + @author = find_author_by_any_email end end diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb index eb2ff0428f6..4b50c3e9c03 100644 --- a/app/models/concerns/mentionable.rb +++ b/app/models/concerns/mentionable.rb @@ -14,6 +14,10 @@ module Mentionable attr = attr.to_s mentionable_attrs << [attr, options] end + + def mentionable_options_for(attr) + mentionable_attrs.detect { |attribute, _options| attribute.to_sym == attr }.try(:last) || {} + end end included do @@ -63,8 +67,8 @@ module Mentionable # Extract GFM references to other Mentionables from this Mentionable. Always excludes its #local_reference. def referenced_mentionables(current_user = self.author) - refs = all_references(current_user) - refs = (refs.issues + refs.merge_requests + refs.commits) + extractor = all_references(current_user) + refs = (extractor.issues + extractor.merge_requests + extractor.commits) # We're using this method instead of Array diffing because that requires # both of the object's `hash` values to be the same, which may not be the @@ -73,8 +77,8 @@ module Mentionable end # Create a cross-reference Note for each GFM reference to another Mentionable found in the +mentionable_attrs+. - def create_cross_references!(author = self.author, without = []) - refs = referenced_mentionables(author) + def create_cross_references!(author = self.author, without = [], refs: nil) + refs ||= referenced_mentionables(author) # We're using this method instead of Array diffing because that requires # both of the object's `hash` values to be the same, which may not be the diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb index c499427605a..f3fb7232242 100644 --- a/app/services/git_push_service.rb +++ b/app/services/git_push_service.rb @@ -106,35 +106,43 @@ class GitPushService < BaseService # Extract any GFM references from the pushed commit messages. If the configured issue-closing regex is matched, # close the referenced Issue. Create cross-reference Notes corresponding to any other referenced Mentionables. def process_commit_messages - is_default_branch = is_default_branch? + closed_issues = commits_close_issues! - authors = Hash.new do |hash, commit| - email = commit.author_email - next hash[email] if hash.has_key?(email) + # Exclude any mentioned Issues to be closed from cross-referencing even if the commits are being pushed to + # a different branch. + commits_cross_references!(closed_issues) + end - hash[email] = commit_user(commit) - end + def commits_close_issues! + # Keep track of the issues that will be actually closed because they are on a default branch. + # Hence, when creating cross-reference notes, the not-closed issues (on non-default branches) + # will also have cross-reference. + closed_issues = Hash.new { |h, k| h[k] = [] } + return closed_issues unless is_default_branch? @push_commits.each do |commit| - # Keep track of the issues that will be actually closed because they are on a default branch. - # Hence, when creating cross-reference notes, the not-closed issues (on non-default branches) - # will also have cross-reference. - closed_issues = [] - - if is_default_branch - # Close issues if these commits were pushed to the project's default branch and the commit message matches the - # closing regex. Exclude any mentioned Issues from cross-referencing even if the commits are being pushed to - # a different branch. - closed_issues = commit.closes_issues(current_user) - closed_issues.each do |issue| - if can?(current_user, :update_issue, issue) - Issues::CloseService.new(project, authors[commit], {}).execute(issue, commit: commit) - end + # Close issues if these commits were pushed to the project's default branch and the commit message matches the + # closing regex. + closed_issues[commit] = commit.closes_issues(current_user) + closed_issues[commit].each do |issue| + if can?(current_user, :update_issue, issue) + Issues::CloseService.new(project, commit_user(commit), {}).execute(issue, commit: commit) end end + end + + closed_issues + end + + def commits_cross_references!(without) + push_extractor = Gitlab::CrossReferenceExtractor.new(project, current_user) + push_extractor.references_with_object(@push_commits, :safe_message) do |commit, refs| + next if refs.empty? - commit.create_cross_references!(authors[commit], closed_issues) - update_issue_metrics(commit, authors) + referenced_issues = refs.select { |ref| ref.is_a?(Issue) } + update_issue_metrics(referenced_issues, commit.committed_date) + + commit.create_cross_references!(authors[commit], without[commit], refs: refs) end end @@ -180,6 +188,7 @@ class GitPushService < BaseService (Gitlab::Git.ref_name(params[:ref]) == project.default_branch || project.default_branch.nil?) end + # Performance is not affected for long list of commits because by default RequestStore is on def commit_user(commit) commit.author || current_user end @@ -188,10 +197,8 @@ class GitPushService < BaseService @branch_name ||= Gitlab::Git.ref_name(params[:ref]) end - def update_issue_metrics(commit, authors) - mentioned_issues = commit.all_references(authors[commit]).issues - - Issue::Metrics.where(issue_id: mentioned_issues.map(&:id), first_mentioned_in_commit_at: nil). - update_all(first_mentioned_in_commit_at: commit.committed_date) + def update_issue_metrics(referenced_issues, referenced_at) + Issue::Metrics.where(issue_id: referenced_issues.map(&:id), first_mentioned_in_commit_at: nil). + update_all(first_mentioned_in_commit_at: referenced_at) end end diff --git a/lib/banzai/mentionable_renderer.rb b/lib/banzai/mentionable_renderer.rb new file mode 100644 index 00000000000..c8c37c6910d --- /dev/null +++ b/lib/banzai/mentionable_renderer.rb @@ -0,0 +1,9 @@ +module Banzai + module MentionableRenderer + def self.render_objects(objects, attr, project, user) + renderer = ObjectRenderer.new(project, user) + + renderer.render_objects(objects, attr) + end + end +end diff --git a/lib/banzai/note_renderer.rb b/lib/banzai/note_renderer.rb index bab6a9934d1..dbcc0d3bf6d 100644 --- a/lib/banzai/note_renderer.rb +++ b/lib/banzai/note_renderer.rb @@ -13,8 +13,7 @@ module Banzai user, requested_path: path, project_wiki: wiki, - ref: git_ref, - pipeline: :note) + ref: git_ref) renderer.render(notes, :note) end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 9aef807c152..ffd11e2425b 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -51,9 +51,31 @@ module Banzai redactor.redact(documents) end + # Renders the attributes of a set of objects. + # + # Returns an Array of `Nokogiri::HTML::Document`. + def render_attributes(objects, attribute) + strings_and_contexts = populate_contexts(objects, attribute) + + Banzai.cache_collection_render(strings_and_contexts).each_with_index.map do |html, index| + Banzai::Pipeline[:relative_link].to_document(html, strings_and_contexts[index][:context]) + end + end + + def populate_contexts(objects, attribute) + objects.map do |object| + context = context_for(object, attribute) + + string = object.__send__(attribute) + + { text: string, context: context } + end + end + # Returns a Banzai context for the given object and attribute. def context_for(object, attribute) - context = base_context.merge(cache_key: [object, attribute]) + context = base_context.merge(base_context_klass_attr(object.class, attribute)) + context[:cache_key] = [object, attribute] if object.respond_to?(:author) context[:author] = object.author @@ -62,20 +84,14 @@ module Banzai context end - # Renders the attributes of a set of objects. - # - # Returns an Array of `Nokogiri::HTML::Document`. - def render_attributes(objects, attribute) - strings_and_contexts = objects.map do |object| - context = context_for(object, attribute) - - string = object.__send__(attribute) + def base_context_klass_attr(klass, attribute) + @base_context_klass_attr ||= Hash.new { |h, k| h[k] = {} } + @base_context_klass_attr[klass][attribute] ||= begin + return {} unless klass.respond_to?(:mentionable_attrs) - { text: string, context: context } - end + _attr, context_klass_options = klass.mentionable_attrs.detect { |attr, _options| attr.to_sym == attribute } - Banzai.cache_collection_render(strings_and_contexts).each_with_index.map do |html, index| - Banzai::Pipeline[:relative_link].to_document(html, strings_and_contexts[index][:context]) + context_klass_options || {} end end diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index 0df3a72d1c4..edf38e6e067 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -19,7 +19,7 @@ module Banzai # # Returns the documents passed as the first argument. def redact(documents) - all_document_nodes = document_nodes(documents) + all_document_nodes = ReferenceQuerying.document_nodes(documents) redact_document_nodes(all_document_nodes) end @@ -28,13 +28,13 @@ module Banzai # # data - An Array of a Hashes mapping an HTML document to nodes to redact. def redact_document_nodes(all_document_nodes) - all_nodes = all_document_nodes.map { |x| x[:nodes] }.flatten + all_nodes = all_document_nodes.map { |x| x.nodes }.flatten visible = nodes_visible_to_user(all_nodes) metadata = [] all_document_nodes.each do |entry| - nodes_for_document = entry[:nodes] - doc_data = { document: entry[:document], visible_reference_count: nodes_for_document.count } + nodes_for_document = entry.nodes + doc_data = { document: entry.document, visible_reference_count: nodes_for_document.count } metadata << doc_data nodes_for_document.each do |node| @@ -72,11 +72,5 @@ module Banzai visible end - - def document_nodes(documents) - documents.map do |document| - { document: document, nodes: Querying.css(document, 'a.gfm[data-reference-type]') } - end - end end end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index f5d110e987b..05f390a9bf8 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -36,6 +36,8 @@ module Banzai attr_accessor :reference_type end + attr_reader :current_user + # Returns the attribute name containing the value for every object to be # parsed by the current parser. # @@ -196,9 +198,9 @@ module Banzai end # Gathers the references for the given HTML nodes. - def gather_references(nodes) - nodes = nodes_user_can_reference(current_user, nodes) - nodes = nodes_visible_to_user(current_user, nodes) + def gather_references(nodes, user = self.current_user) + nodes = nodes_user_can_reference(user, nodes) + nodes = nodes_visible_to_user(user, nodes) referenced_by(nodes) end @@ -224,7 +226,7 @@ module Banzai private - attr_reader :current_user, :project + attr_reader :project def lazy(&block) Gitlab::Lazy.new(&block) diff --git a/lib/banzai/reference_querying.rb b/lib/banzai/reference_querying.rb new file mode 100644 index 00000000000..37b7f622c36 --- /dev/null +++ b/lib/banzai/reference_querying.rb @@ -0,0 +1,37 @@ +module Banzai + class ReferenceQuerying + def self.document_nodes(documents, types = []) + documents.map { |document| DocumentNodes.new(document, types) } + end + + class DocumentNodes + def initialize(document, types = []) + @document = document + @types = types + end + + attr_reader :document, :types + + def nodes + types.empty? ? raw_nodes : nodes_by_type.values.flatten + end + + def nodes_by_type + @nodes_by_type ||= begin + per_type = Hash.new { |hash, key| hash[key] = [] } + raw_nodes.group_by { |node| node.attr('data-reference-type') }.each do |type, nodes| + type_sym = type.to_sym + per_type[type_sym] = nodes if types.include?(type_sym) + end + per_type + end + end + + private + + def raw_nodes + @raw_nodes ||= Querying.css(document, 'a.gfm[data-reference-type]') + end + end + end +end diff --git a/lib/gitlab/cross_reference_extractor.rb b/lib/gitlab/cross_reference_extractor.rb new file mode 100644 index 00000000000..9dda6f6f298 --- /dev/null +++ b/lib/gitlab/cross_reference_extractor.rb @@ -0,0 +1,63 @@ +module Gitlab + class CrossReferenceExtractor + def initialize(project, user) + @project = project + @user = user + end + + def references_with_object(objects, attr) + documents = Banzai::MentionableRenderer.render_objects(objects, attr, project, user) + all_document_nodes = Banzai::ReferenceQuerying.document_nodes(documents, TYPES) + populate_reference_parser_cache(all_document_nodes) + + objects.each_with_index do |object, index| + document_nodes = all_document_nodes[index] + + # Here we're going to limit the references from those only on the commit nodes + # and that are visible for the commit author. + # Using the same parser we ensure the entities are already cached. + refs = TYPES.flat_map { |type| redact_references(type, object, document_nodes) }.compact + + yield object, refs + end + end + + private + + attr_reader :project, :user + + TYPES = %i(issue external_issue merge_request commit) + + def redact_references(type, object, document_nodes) + nodes = document_nodes.nodes_by_type[type] || [] + + refs = parsers[type].gather_references(nodes, object_author(object)).to_a + # From mentionable#referenced_mentionables + refs.reject! { |ref| ref == object.local_reference } + refs + end + + def object_author(object) + object.author || user + end + + # This method find reference using all the nodes to populate internal reference cache on ther Parser classes + # See Banzai::Reference::BaseParser#collection_objects_for_ids + def populate_reference_parser_cache(all_document_nodes) + TYPES.each do |type| + nodes = all_document_nodes.flat_map { |document_nodes| document_nodes.nodes_by_type[type] } + parsers[type].referenced_by(nodes).to_a + end + end + + # We cache parsers because we pretend use their internal caching inverting their execution + # order. + # 1. we get the references on the whole set of nodes + # 2. we get visible_nodes for the user generated nodes + def parsers + @parsers ||= Hash.new do |hash, type| + hash[type] = Banzai::ReferenceParser[type].new(project, user) + end + end + end +end diff --git a/spec/lib/banzai/note_renderer_spec.rb b/spec/lib/banzai/note_renderer_spec.rb index 98f76f36fd5..49556074278 100644 --- a/spec/lib/banzai/note_renderer_spec.rb +++ b/spec/lib/banzai/note_renderer_spec.rb @@ -12,8 +12,7 @@ describe Banzai::NoteRenderer do with(project, user, requested_path: 'foo', project_wiki: wiki, - ref: 'bar', - pipeline: :note). + ref: 'bar'). and_call_original expect_any_instance_of(Banzai::ObjectRenderer). diff --git a/spec/lib/banzai/object_renderer_spec.rb b/spec/lib/banzai/object_renderer_spec.rb index bcdb95250ca..1650984f525 100644 --- a/spec/lib/banzai/object_renderer_spec.rb +++ b/spec/lib/banzai/object_renderer_spec.rb @@ -87,6 +87,37 @@ describe Banzai::ObjectRenderer do expect(context.key?(:author)).to eq(false) end end + + context 'when the object has mentionable attributes' do + let(:object_klass) do + Class.new do + include Mentionable + attr_mentionable :note, pipeline: :single_line + attr_mentionable :message + + attr_reader :note, :message + end + end + let(:object) { object_klass.new } + + it 'includes mentionable attribute options' do + context = renderer.context_for(object, :note) + + expect(context[:pipeline]).to eq(:single_line) + end + + it 'does not include anything extra when mentionable attribute has no options' do + context = renderer.context_for(object, :message) + + expect(context.has_key?(:pipeline)).to eq(false) + end + + it 'does not include anything extra when is not a mentionable attribute' do + context = renderer.context_for(object, :wadus) + + expect(context.has_key?(:pipeline)).to eq(false) + end + end end describe '#render_attributes' do @@ -124,7 +155,7 @@ describe Banzai::ObjectRenderer do describe '#base_context' do let(:context) do - described_class.new(project, user, pipeline: :note).base_context + described_class.new(project, user, wadus: :foo).base_context end it 'returns a Hash' do @@ -132,7 +163,7 @@ describe Banzai::ObjectRenderer do end it 'includes the custom attributes' do - expect(context[:pipeline]).to eq(:note) + expect(context[:wadus]).to eq(:foo) end it 'includes the current user' do diff --git a/spec/lib/banzai/redactor_spec.rb b/spec/lib/banzai/redactor_spec.rb index 254657a881d..180ddc2c98b 100644 --- a/spec/lib/banzai/redactor_spec.rb +++ b/spec/lib/banzai/redactor_spec.rb @@ -30,7 +30,7 @@ describe Banzai::Redactor do doc2_html = '<a class="gfm" data-reference-type="issue">bar</a>' doc2 = Nokogiri::HTML.fragment(doc2_html) - nodes = redactor.document_nodes([doc1, doc2]).map { |x| x[:nodes] } + nodes = Banzai::ReferenceQuerying.document_nodes([doc1, doc2]).map(&:nodes) expect(redactor).to receive(:nodes_visible_to_user).and_return(nodes.flatten) redacted_data = redactor.redact([doc1, doc2]) @@ -51,7 +51,7 @@ describe Banzai::Redactor do with([node]). and_return(Set.new) - redactor.redact_document_nodes([{ document: doc, nodes: [node] }]) + redactor.redact_document_nodes([double(document: doc, nodes: [node])]) expect(doc.to_html).to eq('foo') end diff --git a/spec/lib/gitlab/cross_reference_extractor_spec.rb b/spec/lib/gitlab/cross_reference_extractor_spec.rb new file mode 100644 index 00000000000..da5c48394e2 --- /dev/null +++ b/spec/lib/gitlab/cross_reference_extractor_spec.rb @@ -0,0 +1,54 @@ +require 'spec_helper' + +describe Gitlab::CrossReferenceExtractor, lib: true do + context "as subtitute for mentionable commits" do + # It's shared with the mentionable context should be other name + subject { create(:project).commit } + + include_context 'mentionable context' + + describe '#references_by_object' do + let(:author) { create(:user, email: subject.author_email) } + let(:backref_text) { "commit #{subject.id}" } + let(:set_mentionable_text) do + ->(txt) { allow(subject).to receive(:safe_message).and_return(txt) } + end + + # Include the subject in the repository stub. + let(:extra_commits) { [subject] } + + let(:current_user) { author } + + it "extracts references from its reference property" do + described_class.new(project, current_user).references_with_object([subject], :safe_message) do |object, refs| + # De-duplicate and omit itself + expect(object).to eq(subject) + expect(refs.size).to eq(6) + expect(refs).to include(mentioned_issue) + expect(refs).to include(mentioned_mr) + expect(refs).to include(mentioned_commit) + expect(refs).to include(ext_issue) + expect(refs).to include(ext_mr) + expect(refs).to include(ext_commit) + end + end + + it "uses object author to redact references" do + unknown_author = create(:user) + + expect(subject).to receive(:author).and_return(unknown_author).at_least(:once) + + described_class.new(project, current_user).references_with_object([subject], :safe_message) do |object, refs| + expect(object).to eq(subject) + expect(refs.size).to eq(5) + expect(refs).not_to include(mentioned_issue) + expect(refs).to include(mentioned_mr) + expect(refs).to include(mentioned_commit) + expect(refs).to include(ext_issue) + expect(refs).to include(ext_mr) + expect(refs).to include(ext_commit) + end + end + end + end +end diff --git a/spec/models/commit_spec.rb b/spec/models/commit_spec.rb index d3e6a6648cc..95054228d72 100644 --- a/spec/models/commit_spec.rb +++ b/spec/models/commit_spec.rb @@ -21,7 +21,7 @@ describe Commit, models: true do it 'caches the author' do user = create(:user, email: commit.author_email) - expect(RequestStore).to receive(:active?).twice.and_return(true) + expect(RequestStore).to receive(:active?).and_return(true) expect_any_instance_of(Commit).to receive(:find_author_by_any_email).and_call_original expect(commit.author).to eq(user) diff --git a/spec/models/concerns/mentionable_spec.rb b/spec/models/concerns/mentionable_spec.rb index 132858950d5..320d59a37df 100644 --- a/spec/models/concerns/mentionable_spec.rb +++ b/spec/models/concerns/mentionable_spec.rb @@ -4,8 +4,9 @@ describe Mentionable do class Example include Mentionable - attr_accessor :project, :message + attr_accessor :project, :message, :safe_message attr_mentionable :message + attr_mentionable :safe_message, pipeline: :single_line def author nil @@ -21,9 +22,26 @@ describe Mentionable do mentionable.project = project mentionable.message = 'JIRA-123' + mentionable.safe_message = 'JIRA-123' expect(mentionable.referenced_mentionables).to be_empty end end + + describe '::mentionable_options_for' do + subject { Example } + + it 'returns the options' do + expect(subject.mentionable_options_for(:safe_message)).to eq(pipeline: :single_line) + end + + it 'returns empty hash when no options specified' do + expect(subject.mentionable_options_for(:message)).to eq({}) + end + + it 'returns empty hash when no a mentionable attribute' do + expect(subject.mentionable_options_for(:unknwon_attr)).to eq({}) + end + end end describe Issue, "Mentionable" do diff --git a/spec/support/mentionable_shared_examples.rb b/spec/support/mentionable_shared_examples.rb index f57c82809a6..4d443b92dd8 100644 --- a/spec/support/mentionable_shared_examples.rb +++ b/spec/support/mentionable_shared_examples.rb @@ -9,7 +9,7 @@ shared_context 'mentionable context' do let(:author) { subject.author } let(:mentioned_issue) { create(:issue, project: project) } - let!(:mentioned_mr) { create(:merge_request, source_project: project) } + let(:mentioned_mr) { create(:merge_request, source_project: project) } let(:mentioned_commit) { project.commit("HEAD~1") } let(:ext_proj) { create(:project, :public) } |