diff options
Diffstat (limited to 'lib')
42 files changed, 446 insertions, 183 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e35b1a0ff63..8aad320e376 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -928,7 +928,7 @@ module API end class Tag < Grape::Entity - expose :name, :message + expose :name, :message, :target expose :commit, using: Entities::Commit do |repo_tag, options| options[:project].repository.commit(repo_tag.dereferenced_target) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 3264a26f7d2..d4cc18f622b 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -189,7 +189,7 @@ module API post ":id/merge_requests" do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42316') - authorize! :create_merge_request, user_project + authorize! :create_merge_request_from, user_project mr_params = declared_params(include_missing: false) mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) diff --git a/lib/api/projects.rb b/lib/api/projects.rb index d0a4a23e074..3ae6fbd1fa9 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -338,6 +338,11 @@ module API end end + desc 'Get languages in project repository' + get ':id/languages' do + user_project.repository.languages.map { |language| language.values_at(:label, :value) }.to_h + end + desc 'Remove a project' delete ":id" do authorize! :remove_project, user_project diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index ce216497996..9b0f70e2bfe 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -93,7 +93,7 @@ module API post ":id/merge_requests" do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42126') - authorize! :create_merge_request, user_project + authorize! :create_merge_request_from, user_project mr_params = declared_params(include_missing: false) mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? diff --git a/lib/banzai/commit_renderer.rb b/lib/banzai/commit_renderer.rb index f5ff95e3eb3..c351a155ae5 100644 --- a/lib/banzai/commit_renderer.rb +++ b/lib/banzai/commit_renderer.rb @@ -3,7 +3,7 @@ module Banzai ATTRIBUTES = [:description, :title].freeze def self.render(commits, project, user = nil) - obj_renderer = ObjectRenderer.new(project, user) + obj_renderer = ObjectRenderer.new(user: user, default_project: project) ATTRIBUTES.each { |attr| obj_renderer.render(commits, attr) } end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index a848154b2d4..60a12dca9d3 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -56,29 +56,29 @@ module Banzai # Implement in child class # Example: project.merge_requests.find - def find_object(project, id) + def find_object(parent_object, id) end # Override if the link reference pattern produces a different ID (global # ID vs internal ID, for instance) to the regular reference pattern. - def find_object_from_link(project, id) - find_object(project, id) + def find_object_from_link(parent_object, id) + find_object(parent_object, id) end # Implement in child class # Example: project_merge_request_url - def url_for_object(object, project) + def url_for_object(object, parent_object) end - def find_object_cached(project, id) - cached_call(:banzai_find_object, id, path: [object_class, project.id]) do - find_object(project, id) + def find_object_cached(parent_object, id) + cached_call(:banzai_find_object, id, path: [object_class, parent_object.id]) do + find_object(parent_object, id) end end - def find_object_from_link_cached(project, id) - cached_call(:banzai_find_object_from_link, id, path: [object_class, project.id]) do - find_object_from_link(project, id) + def find_object_from_link_cached(parent_object, id) + cached_call(:banzai_find_object_from_link, id, path: [object_class, parent_object.id]) do + find_object_from_link(parent_object, id) end end @@ -88,9 +88,9 @@ module Banzai end end - def url_for_object_cached(object, project) - cached_call(:banzai_url_for_object, object, path: [object_class, project.id]) do - url_for_object(object, project) + def url_for_object_cached(object, parent_object) + cached_call(:banzai_url_for_object, object, path: [object_class, parent_object.id]) do + url_for_object(object, parent_object) end end diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index 99fa2d9d8fb..01b3b0dafb9 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -23,6 +23,8 @@ module Banzai end def find_object(project, id) + return unless project.is_a?(Project) + range = CommitRange.new(id, project) range.valid_commits? ? range : nil diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 43bf4fc6565..8cd92a1adba 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -17,6 +17,8 @@ module Banzai end def find_object(project, id) + return unless project.is_a?(Project) + if project && project.valid_repo? # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/43894 Gitlab::GitalyClient.allow_n_plus_1_calls { project.commit(id) } diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index 8f541dcfdb2..1a415232545 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -11,7 +11,8 @@ module Banzai def call return doc unless context[:issuable_state_filter_enabled] - extractor = Banzai::IssuableExtractor.new(project, current_user) + context = RenderContext.new(project, current_user) + extractor = Banzai::IssuableExtractor.new(context) issuables = extractor.extract([doc]) issuables.each do |node, issuable| diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 1cbada818fb..a5f38046a43 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -8,8 +8,8 @@ module Banzai Label end - def find_object(project, id) - find_labels(project).find(id) + def find_object(parent_object, id) + find_labels(parent_object).find(id) end def self.references_in(text, pattern = Label.reference_pattern) diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index 1a1d7dbeb3d..b144bd8cf54 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -12,10 +12,14 @@ module Banzai # 'regular' references, we need to use the global ID to disambiguate # between group and project milestones. def find_object(project, id) + return unless project.is_a?(Project) + find_milestone_with_finder(project, id: id) end def find_object_from_link(project, iid) + return unless project.is_a?(Project) + find_milestone_with_finder(project, iid: iid) end @@ -40,7 +44,7 @@ module Banzai project_path = full_project_path(namespace_ref, project_ref) project = parent_from_ref(project_path) - return unless project + return unless project && project.is_a?(Project) milestone_params = milestone_params(milestone_id, milestone_name) diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index 9f9882b3b40..caf11fe94c4 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -7,7 +7,11 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - Redactor.new(project, current_user).redact([doc]) unless context[:skip_redaction] + unless context[:skip_redaction] + context = RenderContext.new(project, current_user) + + Redactor.new(context).redact([doc]) + end doc end diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index 134a192c22b..881e10afb9f 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -12,6 +12,8 @@ module Banzai end def find_object(project, id) + return unless project.is_a?(Project) + project.snippets.find_by(id: id) end diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index 49603d0b363..ae7dc71e7eb 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -12,11 +12,11 @@ module Banzai [@data-reference-type="issue" or @data-reference-type="merge_request"] ).freeze - attr_reader :project, :user + attr_reader :context - def initialize(project, user) - @project = project - @user = user + # context - An instance of Banzai::RenderContext. + def initialize(context) + @context = context end # Returns Hash in the form { node => issuable_instance } @@ -25,8 +25,10 @@ module Banzai document.xpath(QUERY) end - issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user) - merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user) + issue_parser = Banzai::ReferenceParser::IssueParser.new(context) + + merge_request_parser = + Banzai::ReferenceParser::MergeRequestParser.new(context) issuables_for_nodes = issue_parser.records_for_nodes(nodes).merge( merge_request_parser.records_for_nodes(nodes) diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 2691be81623..a176f1e261b 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -13,14 +13,13 @@ module Banzai # As an example, rendering the attribute `note` would place the unredacted # HTML into `note_html` and the redacted HTML into `redacted_note_html`. class ObjectRenderer - attr_reader :project, :user + attr_reader :context - # project - A Project to use for redacting Markdown. + # default_project - A default Project to use for redacting Markdown. # user - The user viewing the Markdown/HTML documents, if any. # redaction_context - A Hash containing extra attributes to use during redaction - def initialize(project, user = nil, redaction_context = {}) - @project = project - @user = user + def initialize(default_project: nil, user: nil, redaction_context: {}) + @context = RenderContext.new(default_project, user) @redaction_context = base_context.merge(redaction_context) end @@ -48,17 +47,21 @@ module Banzai pipeline = HTML::Pipeline.new([]) objects.map do |object| - pipeline.to_document(Banzai.render_field(object, attribute)) + document = pipeline.to_document(Banzai.render_field(object, attribute)) + + context.associate_document(document, object) + + document end end def post_process_documents(documents, objects, attribute) # Called here to populate cache, refer to IssuableExtractor docs - IssuableExtractor.new(project, user).extract(documents) + IssuableExtractor.new(context).extract(documents) documents.zip(objects).map do |document, object| - context = context_for(object, attribute) - Banzai::Pipeline[:post_process].to_document(document, context) + pipeline_context = context_for(document, object, attribute) + Banzai::Pipeline[:post_process].to_document(document, pipeline_context) end end @@ -66,20 +69,21 @@ module Banzai # # Returns an Array containing the redacted documents. def redact_documents(documents) - redactor = Redactor.new(project, user) + redactor = Redactor.new(context) redactor.redact(documents) end # Returns a Banzai context for the given object and attribute. - def context_for(object, attribute) - @redaction_context.merge(object.banzai_render_context(attribute)) + def context_for(document, object, attribute) + @redaction_context.merge(object.banzai_render_context(attribute)).merge( + project: context.project_for_node(document) + ) end def base_context { - current_user: user, - project: project, + current_user: context.current_user, skip_redaction: true } end diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index fd457bebf03..28928d6f376 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -2,13 +2,15 @@ module Banzai # Class for removing Markdown references a certain user is not allowed to # view. class Redactor - attr_reader :user, :project + attr_reader :context - # project - A Project to use for redacting links. - # user - The currently logged in user (if any). - def initialize(project, user = nil) - @project = project - @user = user + # context - An instance of `Banzai::RenderContext`. + def initialize(context) + @context = context + end + + def user + context.current_user end # Redacts the references in the given Array of documents. @@ -70,11 +72,11 @@ module Banzai end def redact_cross_project_references(documents) - extractor = Banzai::IssuableExtractor.new(project, user) + extractor = Banzai::IssuableExtractor.new(context) issuables = extractor.extract(documents) issuables.each do |node, issuable| - next if issuable.project == project + next if issuable.project == context.project_for_node(node) node['class'] = node['class'].gsub('has-tooltip', '') node['title'] = nil @@ -95,7 +97,7 @@ module Banzai end per_type.each do |type, nodes| - parser = Banzai::ReferenceParser[type].new(project, user) + parser = Banzai::ReferenceParser[type].new(context) visible.merge(parser.nodes_visible_to_user(user, nodes)) end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index 7e6357f8a00..78588299c18 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -10,8 +10,8 @@ module Banzai end def references(type, project, current_user = nil) - processor = Banzai::ReferenceParser[type] - .new(project, current_user) + context = RenderContext.new(project, current_user) + processor = Banzai::ReferenceParser[type].new(context) processor.process(html_documents) end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 279fca8d043..68752f5bb5a 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -45,9 +45,13 @@ module Banzai @data_attribute ||= "data-#{reference_type.to_s.dasherize}" end - def initialize(project = nil, current_user = nil) - @project = project - @current_user = current_user + # context - An instance of `Banzai::RenderContext`. + def initialize(context) + @context = context + end + + def project_for_node(node) + context.project_for_node(node) end # Returns all the nodes containing references that the user can refer to. @@ -224,7 +228,11 @@ module Banzai private - attr_reader :current_user, :project + attr_reader :context + + def current_user + context.current_user + end # When a feature is disabled or visible only for # team members we should not allow team members diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb index a50e6f8ef8f..2920e886938 100644 --- a/lib/banzai/reference_parser/commit_range_parser.rb +++ b/lib/banzai/reference_parser/commit_range_parser.rb @@ -29,6 +29,8 @@ module Banzai end def find_object(project, id) + return unless project.is_a?(Project) + range = CommitRange.new(id, project) range.valid_commits? ? range : nil diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 230827129b6..6bee5ea15b9 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -5,15 +5,10 @@ module Banzai def nodes_visible_to_user(user, nodes) issues = records_for_nodes(nodes) - issues_to_check = issues.values + issues_to_check, cross_project_issues = partition_issues(issues, user) - unless can?(user, :read_cross_project) - issues_to_check, cross_project_issues = issues_to_check.partition do |issue| - issue.project == project - end - end - - readable_issues = Ability.issues_readable_by_user(issues_to_check, user).to_set + readable_issues = + Ability.issues_readable_by_user(issues_to_check, user).to_set nodes.select do |node| issue_in_node = issues[node] @@ -25,7 +20,7 @@ module Banzai # but not the issue. if readable_issues.include?(issue_in_node) true - elsif cross_project_issues&.include?(issue_in_node) + elsif cross_project_issues.include?(issue_in_node) can_read_reference?(user, issue_in_node) else false @@ -33,6 +28,32 @@ module Banzai end end + # issues - A Hash mapping HTML nodes to their corresponding Issue + # instances. + # user - The current User. + def partition_issues(issues, user) + return [issues.values, []] if can?(user, :read_cross_project) + + issues_to_check = [] + cross_project_issues = [] + + # We manually partition the data since our input is a Hash and our + # output has to be an Array of issues; not an Array of (node, issue) + # pairs. + issues.each do |node, issue| + target = + if issue.project == project_for_node(node) + issues_to_check + else + cross_project_issues + end + + target << issue + end + + [issues_to_check, cross_project_issues] + end + def records_for_nodes(nodes) @issues_for_nodes ||= grouped_objects_for_nodes( nodes, diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index 8932d4f2905..ceb7f1d165c 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -58,7 +58,7 @@ module Banzai def can_read_project_reference?(node) node_id = node.attr('data-project').to_i - project && project.id == node_id + project_for_node(node)&.id == node_id end def nodes_user_can_reference(current_user, nodes) @@ -71,6 +71,7 @@ module Banzai nodes.select do |node| project_id = node.attr(project_attr) user_id = node.attr(author_attr) + project = project_for_node(node) if project && project_id && project.id == project_id.to_i true diff --git a/lib/banzai/render_context.rb b/lib/banzai/render_context.rb new file mode 100644 index 00000000000..e30fc9f469b --- /dev/null +++ b/lib/banzai/render_context.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Banzai + # Object storing the current user, project, and other details used when + # parsing Markdown references. + class RenderContext + attr_reader :current_user + + # default_project - The default project to use for all documents, if any. + # current_user - The user viewing the document, if any. + def initialize(default_project = nil, current_user = nil) + @current_user = current_user + @projects = Hash.new(default_project) + end + + # Associates an HTML document with a Project. + # + # document - The HTML document to map to a Project. + # object - The object that produced the HTML document. + def associate_document(document, object) + # XML nodes respond to "document" but will return a Document instance, + # even when they belong to a DocumentFragment. + document = document.document if document.fragment? + + @projects[document] = object.project if object.respond_to?(:project) + end + + def project_for_node(node) + @projects[node.document] + end + end +end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index aa9fd36d9ff..37ce01925bb 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -12,4 +12,12 @@ module Gitlab def self.staging? Gitlab.config.gitlab.url == 'https://staging.gitlab.com' end + + def self.dev? + Gitlab.config.gitlab.url == 'https://dev.gitlab.org' + end + + def self.inc_controlled? + dev? || staging? || com? + end end diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index c0c7c7f5b5d..c1fc70ac266 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -3,6 +3,14 @@ module Gitlab module Status module Build module Common + def illustration + { + image: 'illustrations/skipped-job_empty.svg', + size: 'svg-430', + title: _('This job does not have a trace.') + } + end + def has_details? can?(user, :read_build, subject) end diff --git a/lib/gitlab/ci/status/build/erased.rb b/lib/gitlab/ci/status/build/erased.rb index 3a5113b16b6..495227c2ffb 100644 --- a/lib/gitlab/ci/status/build/erased.rb +++ b/lib/gitlab/ci/status/build/erased.rb @@ -5,7 +5,7 @@ module Gitlab class Erased < Status::Extended def illustration { - image: 'illustrations/skipped-job_empty.svg', + image: 'illustrations/erased-log_empty.svg', size: 'svg-430', title: _('Job has been erased') } diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 23ed71db8b0..d00e5b07f95 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -3,12 +3,9 @@ module Gitlab module Variables class Collection class Item - def initialize(**options) + def initialize(key:, value:, public: true, file: false) @variable = { - key: options.fetch(:key), - value: options.fetch(:value), - public: options.fetch(:public, true), - file: options.fetch(:files, false) + key: key, value: value, public: public, file: file } end diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index 3436306e122..2f864f2082b 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -23,7 +23,8 @@ module Gitlab def execute raise ProjectNotFound unless project - validate_permission!(:create_merge_request) + validate_permission!(:create_merge_request_in) + validate_permission!(:create_merge_request_from) verify_record!( record: create_merge_request, diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index d4e893b881c..c9abea90d21 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -1,5 +1,9 @@ module Gitlab module Git + # The ID of empty tree. + # See http://stackoverflow.com/a/40884093/1856239 and + # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 + EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze BLANK_SHA = ('0' * 40).freeze TAG_REF_PREFIX = "refs/tags/".freeze BRANCH_REF_PREFIX = "refs/heads/".freeze diff --git a/lib/gitlab/git/attributes_parser.rb b/lib/gitlab/git/attributes_parser.rb index d8aeabb6cba..08f4d7d4f5c 100644 --- a/lib/gitlab/git/attributes_parser.rb +++ b/lib/gitlab/git/attributes_parser.rb @@ -3,12 +3,8 @@ module Gitlab # Class for parsing Git attribute files and extracting the attributes for # file patterns. class AttributesParser - def initialize(attributes_data) + def initialize(attributes_data = "") @data = attributes_data || "" - - if @data.is_a?(File) - @patterns = parse_file - end end # Returns all the Git attributes for the given path. @@ -28,7 +24,7 @@ module Gitlab # Returns a Hash containing the file patterns and their attributes. def patterns - @patterns ||= parse_file + @patterns ||= parse_data end # Parses an attribute string. @@ -91,8 +87,8 @@ module Gitlab private - # Parses the Git attributes file. - def parse_file + # Parses the Git attributes file contents. + def parse_data pairs = [] comment = '#' diff --git a/lib/gitlab/git/info_attributes.rb b/lib/gitlab/git/info_attributes.rb deleted file mode 100644 index e79a440950b..00000000000 --- a/lib/gitlab/git/info_attributes.rb +++ /dev/null @@ -1,49 +0,0 @@ -# Gitaly note: JV: not sure what to make of this class. Why does it use -# the full disk path of the repository to look up attributes This is -# problematic in Gitaly, because Gitaly hides the full disk path to the -# repository from gitlab-ce. - -module Gitlab - module Git - # Parses gitattributes at `$GIT_DIR/info/attributes` - # - # Unlike Rugged this parser only needs a single IO call (a call to `open`), - # vastly reducing the time spent in extracting attributes. - # - # This class _only_ supports parsing the attributes file located at - # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files - # (`.gitattributes` is copied to this particular path). - # - # Basic usage: - # - # attributes = Gitlab::Git::InfoAttributes.new(some_repo.path) - # - # attributes.attributes('README.md') # => { "eol" => "lf } - class InfoAttributes - delegate :attributes, :patterns, to: :parser - - # path - The path to the Git repository. - def initialize(path) - @repo_path = File.expand_path(path) - end - - def parser - @parser ||= begin - if File.exist?(attributes_path) - File.open(attributes_path, 'r') do |file_handle| - AttributesParser.new(file_handle) - end - else - AttributesParser.new("") - end - end - end - - private - - def attributes_path - @attributes_path ||= File.join(@repo_path, 'info/attributes') - end - end - end -end diff --git a/lib/gitlab/git/raw_diff_change.rb b/lib/gitlab/git/raw_diff_change.rb new file mode 100644 index 00000000000..eb3d8819239 --- /dev/null +++ b/lib/gitlab/git/raw_diff_change.rb @@ -0,0 +1,60 @@ +module Gitlab + module Git + # This class behaves like a struct with fields :blob_id, :blob_size, :operation, :old_path, :new_path + # All those fields are (binary) strings or integers + class RawDiffChange + attr_reader :blob_id, :blob_size, :old_path, :new_path, :operation + + def initialize(raw_change) + parse(raw_change) + end + + private + + # Input data has the following format: + # + # When a file has been modified: + # 7e3e39ebb9b2bf433b4ad17313770fbe4051649c 669 M\tfiles/ruby/popen.rb + # + # When a file has been renamed: + # 85bc2f9753afd5f4fc5d7c75f74f8d526f26b4f3 107 R060\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee + def parse(raw_change) + @blob_id, @blob_size, @raw_operation, raw_paths = raw_change.split(' ', 4) + @operation = extract_operation + @old_path, @new_path = extract_paths(raw_paths) + end + + def extract_paths(file_path) + case operation + when :renamed + file_path.split(/\t/) + when :deleted + [file_path, nil] + when :added + [nil, file_path] + else + [file_path, file_path] + end + end + + def extract_operation + case @raw_operation&.first(1) + when 'A' + :added + when 'C' + :copied + when 'D' + :deleted + when 'M' + :modified + when 'R' + :renamed + when 'T' + :type_changed + else + :unknown + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index f1b575bd872..11a421cb430 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -75,9 +75,6 @@ module Gitlab end end - # Full path to repo - attr_reader :path - # Directory name of repo attr_reader :name @@ -96,22 +93,26 @@ module Gitlab @relative_path = relative_path @gl_repository = gl_repository - storage_path = Gitlab.config.repositories.storages[@storage].legacy_disk_path @gitlab_projects = Gitlab::Git::GitlabProjects.new( storage, relative_path, global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, logger: Rails.logger ) - @path = File.join(storage_path, @relative_path) + @name = @relative_path.split("/").last - @attributes = Gitlab::Git::InfoAttributes.new(path) end def ==(other) path == other.path end + def path + @path ||= File.join( + Gitlab.config.repositories.storages[@storage].legacy_disk_path, @relative_path + ) + end + # Default branch in the repository def root_ref @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| @@ -140,12 +141,12 @@ module Gitlab end def exists? - Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| + Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_repository_client.exists? else circuit_breaker.perform do - File.exist?(File.join(@path, 'refs')) + File.exist?(File.join(path, 'refs')) end end end @@ -559,6 +560,24 @@ module Gitlab count_commits(from: from, to: to, **options) end + # old_rev and new_rev are commit ID's + # the result of this method is an array of Gitlab::Git::RawDiffChange + def raw_changes_between(old_rev, new_rev) + result = [] + + circuit_breaker.perform do + Open3.pipeline_r(git_diff_cmd(old_rev, new_rev), format_git_cat_file_script, git_cat_file_cmd) do |last_stdout, wait_threads| + last_stdout.each_line { |line| result << ::Gitlab::Git::RawDiffChange.new(line.chomp!) } + + if wait_threads.any? { |waiter| !waiter.value&.success? } + raise ::Gitlab::Git::Repository::GitError, "Unabled to obtain changes between #{old_rev} and #{new_rev}" + end + end + end + + result + end + # Returns the SHA of the most recent common ancestor of +from+ and +to+ def merge_base(from, to) gitaly_migrate(:merge_base) do |is_enabled| @@ -993,11 +1012,32 @@ module Gitlab raise InvalidRef end + def info_attributes + return @info_attributes if @info_attributes + + content = + gitaly_migrate(:get_info_attributes) do |is_enabled| + if is_enabled + gitaly_repository_client.info_attributes + else + attributes_path = File.join(File.expand_path(path), 'info', 'attributes') + + if File.exist?(attributes_path) + File.read(attributes_path) + else + "" + end + end + end + + @info_attributes = AttributesParser.new(content) + end + # Returns the Git attributes for the given file path. # # See `Gitlab::Git::Attributes` for more information. def attributes(path) - @attributes.attributes(path) + info_attributes.attributes(path) end def gitattribute(path, name) @@ -1385,7 +1425,8 @@ module Gitlab end def branch_names_contains_sha(sha) - gitaly_migrate(:branch_names_contains_sha) do |is_enabled| + gitaly_migrate(:branch_names_contains_sha, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_ref_client.branch_names_contains_sha(sha) else @@ -1395,7 +1436,8 @@ module Gitlab end def tag_names_contains_sha(sha) - gitaly_migrate(:tag_names_contains_sha) do |is_enabled| + gitaly_migrate(:tag_names_contains_sha, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_ref_client.tag_names_contains_sha(sha) else @@ -1428,7 +1470,7 @@ module Gitlab return [] if empty? || safe_query.blank? - args = %W(ls-tree --full-tree -r #{ref || root_ref} --name-status | #{safe_query}) + args = %W(ls-tree -r --name-status --full-tree #{ref || root_ref} -- #{safe_query}) run_git(args).first.lines.map(&:strip) end @@ -2461,6 +2503,35 @@ module Gitlab result.to_s(16) end + + def build_git_cmd(*args) + object_directories = alternate_object_directories.join(File::PATH_SEPARATOR) + + env = { 'PWD' => self.path } + env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories if object_directories.present? + + [ + env, + ::Gitlab.config.git.bin_path, + *args, + { chdir: self.path } + ] + end + + def git_diff_cmd(old_rev, new_rev) + old_rev = old_rev == ::Gitlab::Git::BLANK_SHA ? ::Gitlab::Git::EMPTY_TREE_ID : old_rev + + build_git_cmd('diff', old_rev, new_rev, '--raw') + end + + def git_cat_file_cmd + format = '%(objectname) %(objectsize) %(rest)' + build_git_cmd('cat-file', "--batch-check=#{format}") + end + + def format_git_cat_file_script + File.expand_path('../support/format-git-cat-file-input', __FILE__) + end end end end diff --git a/lib/gitlab/git/support/format-git-cat-file-input b/lib/gitlab/git/support/format-git-cat-file-input new file mode 100755 index 00000000000..2e93c646d0f --- /dev/null +++ b/lib/gitlab/git/support/format-git-cat-file-input @@ -0,0 +1,21 @@ +#!/usr/bin/env ruby + +# This script formats the output of the `git diff <old_rev> <new_rev> --raw` +# command so it can be processed by `git cat-file` + +# We need to convert this: +# ":100644 100644 5f53439... 85bc2f9... R060\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee" +# To: +# "85bc2f9 R\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee" + +ARGF.each do |line| + _, _, old_blob_id, new_blob_id, rest = line.split(/\s/, 5) + + old_blob_id.gsub!(/[^\h]/, '') + new_blob_id.gsub!(/[^\h]/, '') + + # We can't pass '0000000...' to `git cat-file` given it will not return info about the deleted file + blob_id = new_blob_id =~ /\A0+\z/ ? old_blob_id : new_blob_id + + $stdout.puts "#{blob_id} #{rest}" +end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 456a8a1a2d6..a36e6c822f9 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -3,10 +3,6 @@ module Gitlab class CommitService include Gitlab::EncodingHelper - # The ID of empty tree. - # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 - EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze - def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository @@ -37,7 +33,7 @@ module Gitlab def diff(from, to, options = {}) from_id = case from when NilClass - EMPTY_TREE_ID + Gitlab::Git::EMPTY_TREE_ID else if from.respond_to?(:oid) # This is meant to match a Rugged::Commit. This should be impossible in @@ -50,7 +46,7 @@ module Gitlab to_id = case to when NilClass - EMPTY_TREE_ID + Gitlab::Git::EMPTY_TREE_ID else if to.respond_to?(:oid) # This is meant to match a Rugged::Commit. This should be impossible in @@ -352,7 +348,7 @@ module Gitlab end def diff_from_parent_request_params(commit, options = {}) - parent_id = commit.parent_ids.first || EMPTY_TREE_ID + parent_id = commit.parent_ids.first || Gitlab::Git::EMPTY_TREE_ID diff_between_commits_request_params(parent_id, commit.id, options) end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 6441065f5fe..39057beefba 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -50,6 +50,15 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) end + def info_attributes + request = Gitaly::GetInfoAttributesRequest.new(repository: @gitaly_repo) + + response = GitalyClient.call(@storage, :repository_service, :get_info_attributes, request) + response.each_with_object("") do |message, attributes| + attributes << message.attributes + end + end + def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:, prune: true) request = Gitaly::FetchRemoteRequest.new( repository: @gitaly_repo, remote: remote, force: forced, diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 0d8dd5cb8f4..7a698e4b3f3 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -136,7 +136,7 @@ module Gitlab wiki_file = nil response.each do |message| - next unless message.name.present? + next unless message.name.present? || wiki_file if wiki_file wiki_file.raw_data << message.raw_data diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 24393f96d96..69952cbb47c 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -51,7 +51,7 @@ module Gitlab return false unless can_access_git? if protected?(ProtectedBranch, project, ref) - user.can?(:delete_protected_branch, project) + user.can?(:push_to_delete_protected_branch, project) else user.can?(:push_code, project) end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index b0a492eaa58..aeda66763e8 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -73,6 +73,10 @@ module Gitlab nil end + def bytes_to_megabytes(bytes) + bytes.to_f / Numeric::MEGABYTE + end + # Used in EE # Accepts either an Array or a String and returns an array def ensure_array_from_string(string_or_array) diff --git a/lib/rspec_flaky/config.rb b/lib/rspec_flaky/config.rb index a17ae55910e..06e96f969f1 100644 --- a/lib/rspec_flaky/config.rb +++ b/lib/rspec_flaky/config.rb @@ -1,9 +1,7 @@ -require 'json' - module RspecFlaky class Config def self.generate_report? - ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true' + !!(ENV['FLAKY_RSPEC_GENERATE_REPORT'] =~ /1|true/) end def self.suite_flaky_examples_report_path diff --git a/lib/rspec_flaky/flaky_examples_collection.rb b/lib/rspec_flaky/flaky_examples_collection.rb index 973c95b0212..dea23c325be 100644 --- a/lib/rspec_flaky/flaky_examples_collection.rb +++ b/lib/rspec_flaky/flaky_examples_collection.rb @@ -1,11 +1,9 @@ -require 'json' +require 'active_support/hash_with_indifferent_access' + +require_relative 'flaky_example' module RspecFlaky class FlakyExamplesCollection < SimpleDelegator - def self.from_json(json) - new(JSON.parse(json)) - end - def initialize(collection = {}) unless collection.is_a?(Hash) raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" @@ -22,7 +20,7 @@ module RspecFlaky super(Hash[collection_of_flaky_examples]) end - def to_report + def to_h Hash[map { |uid, example| [uid, example.to_h] }].deep_symbolize_keys end diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb index 4a5bfec9967..5b5e4f7c7de 100644 --- a/lib/rspec_flaky/listener.rb +++ b/lib/rspec_flaky/listener.rb @@ -1,5 +1,11 @@ require 'json' +require_relative 'config' +require_relative 'example' +require_relative 'flaky_example' +require_relative 'flaky_examples_collection' +require_relative 'report' + module RspecFlaky class Listener # - suite_flaky_examples: contains all the currently tracked flacky example @@ -9,7 +15,7 @@ module RspecFlaky attr_reader :suite_flaky_examples, :flaky_examples def initialize(suite_flaky_examples_json = nil) - @flaky_examples = FlakyExamplesCollection.new + @flaky_examples = RspecFlaky::FlakyExamplesCollection.new @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json) end @@ -18,47 +24,36 @@ module RspecFlaky return unless current_example.attempts > 1 - flaky_example = suite_flaky_examples.fetch(current_example.uid) { FlakyExample.new(current_example) } + flaky_example = suite_flaky_examples.fetch(current_example.uid) { RspecFlaky::FlakyExample.new(current_example) } flaky_example.update_flakiness!(last_attempts_count: current_example.attempts) flaky_examples[current_example.uid] = flaky_example end def dump_summary(_) - write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path) + RspecFlaky::Report.new(flaky_examples).write(RspecFlaky::Config.flaky_examples_report_path) + # write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path) new_flaky_examples = flaky_examples - suite_flaky_examples if new_flaky_examples.any? Rails.logger.warn "\nNew flaky examples detected:\n" - Rails.logger.warn JSON.pretty_generate(new_flaky_examples.to_report) + Rails.logger.warn JSON.pretty_generate(new_flaky_examples.to_h) - write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path) + RspecFlaky::Report.new(new_flaky_examples).write(RspecFlaky::Config.new_flaky_examples_report_path) + # write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path) end end - def to_report(examples) - Hash[examples.map { |k, ex| [k, ex.to_h] }] - end - private def init_suite_flaky_examples(suite_flaky_examples_json = nil) - unless suite_flaky_examples_json + if suite_flaky_examples_json + RspecFlaky::Report.load_json(suite_flaky_examples_json).flaky_examples + else return {} unless File.exist?(RspecFlaky::Config.suite_flaky_examples_report_path) - suite_flaky_examples_json = File.read(RspecFlaky::Config.suite_flaky_examples_report_path) + RspecFlaky::Report.load(RspecFlaky::Config.suite_flaky_examples_report_path).flaky_examples end - - FlakyExamplesCollection.from_json(suite_flaky_examples_json) - end - - def write_report_file(examples_collection, file_path) - return unless RspecFlaky::Config.generate_report? - - report_path_dir = File.dirname(file_path) - FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir) - - File.write(file_path, JSON.pretty_generate(examples_collection.to_report)) end end end diff --git a/lib/rspec_flaky/report.rb b/lib/rspec_flaky/report.rb new file mode 100644 index 00000000000..a8730d3b7c7 --- /dev/null +++ b/lib/rspec_flaky/report.rb @@ -0,0 +1,54 @@ +require 'json' +require 'time' + +require_relative 'config' +require_relative 'flaky_examples_collection' + +module RspecFlaky + # This class is responsible for loading/saving JSON reports, and pruning + # outdated examples. + class Report < SimpleDelegator + OUTDATED_DAYS_THRESHOLD = 90 + + attr_reader :flaky_examples + + def self.load(file_path) + load_json(File.read(file_path)) + end + + def self.load_json(json) + new(RspecFlaky::FlakyExamplesCollection.new(JSON.parse(json))) + end + + def initialize(flaky_examples) + unless flaky_examples.is_a?(RspecFlaky::FlakyExamplesCollection) + raise ArgumentError, "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, #{flaky_examples.class} given!" + end + + @flaky_examples = flaky_examples + super(flaky_examples) + end + + def write(file_path) + unless RspecFlaky::Config.generate_report? + puts "! Generating reports is disabled. To enable it, please set the `FLAKY_RSPEC_GENERATE_REPORT=1` !" # rubocop:disable Rails/Output + return + end + + report_path_dir = File.dirname(file_path) + FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir) + + File.write(file_path, JSON.pretty_generate(flaky_examples.to_h)) + end + + def prune_outdated(days: OUTDATED_DAYS_THRESHOLD) + outdated_date_threshold = Time.now - (3600 * 24 * days) + updated_hash = flaky_examples.dup + .delete_if do |uid, hash| + hash[:last_flaky_at] && Time.parse(hash[:last_flaky_at]).to_i < outdated_date_threshold.to_i + end + + self.class.new(RspecFlaky::FlakyExamplesCollection.new(updated_hash)) + end + end +end |