diff options
author | Felipe Artur <felipefac@gmail.com> | 2019-02-19 11:35:29 -0300 |
---|---|---|
committer | Felipe Artur <felipefac@gmail.com> | 2019-02-19 11:35:29 -0300 |
commit | 648b87315d12d18c92ea14b14ae827480ab3093a (patch) | |
tree | 8c6bab95481bec1c65f0af2fece58b000c3396b7 /lib | |
parent | 52155d8cf8374e9184c2ae834cab761b7520db93 (diff) | |
parent | 0aa64cf80ccd7fda10641af0cd43c4c0a7f3e133 (diff) | |
download | gitlab-ce-648b87315d12d18c92ea14b14ae827480ab3093a.tar.gz |
Merge branch 'master' into issue_51789_part_1
Diffstat (limited to 'lib')
44 files changed, 448 insertions, 126 deletions
diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 9d23daafe95..8defc59224d 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -323,6 +323,22 @@ module API present paginate(commit.merge_requests), with: Entities::MergeRequestBasic end + + desc "Get a commit's GPG signature" do + success Entities::CommitSignature + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end + get ':id/repository/commits/:sha/signature', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do + commit = user_project.commit(params[:sha]) + not_found! 'Commit' unless commit + + signature = commit.signature + not_found! 'GPG Signature' unless signature + + present signature, with: Entities::CommitSignature + end end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index beb8ce349b4..9199f898ea0 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -369,8 +369,9 @@ module API end class Commit < Grape::Entity - expose :id, :short_id, :title, :created_at + expose :id, :short_id, :created_at expose :parent_ids + expose :full_title, as: :title expose :safe_message, as: :message expose :author_name, :author_email, :authored_date expose :committer_name, :committer_email, :committed_date @@ -391,6 +392,13 @@ module API expose :project_id end + class CommitSignature < Grape::Entity + expose :gpg_key_id + expose :gpg_key_primary_keyid, :gpg_key_user_name, :gpg_key_user_email + expose :verification_status + expose :gpg_key_subkey_id + end + class BasicRef < Grape::Entity expose :type, :name end @@ -549,6 +557,15 @@ module API expose :time_stats, using: 'API::Entities::IssuableTimeStats' do |issue| issue end + + expose :merge_requests_count do |issue, options| + if options[:issuable_metadata] + # Avoids an N+1 query when metadata is included + options[:issuable_metadata][issue.id].merge_requests_count + else + issue.merge_requests_closing_issues.count + end + end end class Issue < IssueBasic @@ -731,6 +748,12 @@ module API def build_available?(options) options[:project]&.feature_available?(:builds, options[:current_user]) end + + expose :user do + expose :can_merge do |merge_request, options| + merge_request.can_be_merged_by?(options[:current_user]) + end + end end class MergeRequestChanges < MergeRequest @@ -1003,7 +1026,7 @@ module API end class LabelBasic < Grape::Entity - expose :id, :name, :color, :description + expose :id, :name, :color, :description, :text_color end class Label < LabelBasic @@ -1031,6 +1054,9 @@ module API expose :priority do |label, options| label.priority(options[:parent]) end + expose :is_project_label do |label, options| + label.is_a?(::ProjectLabel) + end end class List < Grape::Entity diff --git a/lib/api/features.rb b/lib/api/features.rb index 835aac05905..4dc1834c644 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -42,6 +42,7 @@ module API requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time' optional :feature_group, type: String, desc: 'A Feature group name' optional :user, type: String, desc: 'A GitLab username' + optional :group, type: String, desc: "A GitLab group's path, such as 'gitlab-org'" optional :project, type: String, desc: 'A projects path, like gitlab-org/gitlab-ce' end post ':name' do diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 2eb7b04711a..54cd4cd9cdb 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -299,6 +299,12 @@ module API items.search(text) end + def order_options_with_tie_breaker + order_options = { params[:order_by] => params[:sort] } + order_options['id'] ||= 'desc' + order_options + end + # error helpers def forbidden!(reason = nil) @@ -393,7 +399,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord def reorder_projects(projects) - projects.reorder(params[:order_by] => params[:sort]) + projects.reorder(order_options_with_tie_breaker) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 4eaaca96b49..fe78049af87 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -81,6 +81,14 @@ module API Gitlab::GlRepository.gl_repository(project, wiki?) end + def gl_project_path + if wiki? + project.wiki.full_path + else + project.full_path + end + end + # Return the repository depending on whether we want the wiki or the # regular repository def repository diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 9488b3469d9..70b32f7d758 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -77,6 +77,7 @@ module API when ::Gitlab::GitAccessResult::Success payload = { gl_repository: gl_repository, + gl_project_path: gl_project_path, gl_id: Gitlab::GlId.gl_id(user), gl_username: user&.username, git_config_options: [], @@ -117,13 +118,7 @@ module API raise ActiveRecord::RecordNotFound.new("No key_id or user_id passed!") end - token_handler = Gitlab::LfsToken.new(actor) - - { - username: token_handler.actor_name, - lfs_token: token_handler.token, - repository_http_path: project.http_url_to_repo - } + Gitlab::LfsToken.new(actor).authentication_payload(project.http_url_to_repo) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/issues.rb b/lib/api/issues.rb index b3636c98550..94ed9ac6fb1 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -29,8 +29,7 @@ module API issues = IssuesFinder.new(current_user, args).execute .preload(:assignees, :labels, :notes, :timelogs, :project, :author, :closed_by) - - issues.reorder(args[:order_by] => args[:sort]) + issues.reorder(order_options_with_tie_breaker) end # rubocop: enable CodeReuse/ActiveRecord @@ -304,19 +303,14 @@ module API get ':id/issues/:issue_iid/related_merge_requests' do issue = find_project_issue(params[:issue_iid]) - merge_request_iids = ::Issues::ReferencedMergeRequestsService.new(user_project, current_user) + merge_requests = ::Issues::ReferencedMergeRequestsService.new(user_project, current_user) .execute(issue) .flatten - .map(&:iid) - - merge_requests = - if merge_request_iids.present? - MergeRequestsFinder.new(current_user, project_id: user_project.id, iids: merge_request_iids).execute - else - MergeRequest.none - end - present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + present paginate(::Kaminari.paginate_array(merge_requests)), + with: Entities::MergeRequestBasic, + current_user: current_user, + project: user_project end desc 'List merge requests closing issue' do diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index df46b4446ff..f8d2ba49d2f 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -38,7 +38,7 @@ module API args[:scope] = args[:scope].underscore if args[:scope] merge_requests = MergeRequestsFinder.new(current_user, args).execute - .reorder(args[:order_by] => args[:sort]) + .reorder(order_options_with_tie_breaker) merge_requests = paginate(merge_requests) .preload(:source_project, :target_project) diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 1bdf7aeb119..f7bd092ce50 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -39,7 +39,7 @@ module API # at the DB query level (which we cannot in that case), the current # page can have less elements than :per_page even if # there's more than one page. - raw_notes = noteable.notes.with_metadata.reorder(params[:order_by] => params[:sort]) + raw_notes = noteable.notes.with_metadata.reorder(order_options_with_tie_breaker) notes = # paginate() only works with a relation. This could lead to a # mismatch between the pagination headers info and the actual notes diff --git a/lib/api/services.rb b/lib/api/services.rb index 637b5a8a89a..145897516a0 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -368,46 +368,9 @@ module API name: :webhook, type: String, desc: 'The Hangouts Chat webhook. e.g. https://chat.googleapis.com/v1/spaces…' - } - ], - 'hipchat' => [ - { - required: true, - name: :token, - type: String, - desc: 'The room token' }, - { - required: false, - name: :room, - type: String, - desc: 'The room name or ID' - }, - { - required: false, - name: :color, - type: String, - desc: 'The room color' - }, - { - required: false, - name: :notify, - type: Boolean, - desc: 'Enable notifications' - }, - { - required: false, - name: :api_version, - type: String, - desc: 'Leave blank for default (v2)' - }, - { - required: false, - name: :server, - type: String, - desc: 'Leave blank for default. https://hipchat.example.com' - } - ], + CHAT_NOTIFICATION_EVENTS + ].flatten, 'irker' => [ { required: true, @@ -468,7 +431,7 @@ module API { required: false, name: :jira_issue_transition_id, - type: Integer, + type: String, desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`' } ], @@ -691,7 +654,6 @@ module API ExternalWikiService, FlowdockService, HangoutsChatService, - HipchatService, IrkerService, JiraService, KubernetesService, diff --git a/lib/api/users.rb b/lib/api/users.rb index 8ce09a8881b..7d88880d412 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -26,7 +26,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord def reorder_users(users) if params[:order_by] && params[:sort] - users.reorder(params[:order_by] => params[:sort]) + users.reorder(order_options_with_tie_breaker) else users end diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index ef0e3decc2c..994074ddc67 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -11,9 +11,7 @@ module API } end - params :wiki_page_params do - requires :content, type: String, desc: 'Content of a wiki page' - requires :title, type: String, desc: 'Title of a wiki page' + params :common_wiki_page_params do optional :format, type: String, values: ProjectWiki::MARKUPS.values.map(&:to_s), @@ -54,7 +52,9 @@ module API success Entities::WikiPage end params do - use :wiki_page_params + requires :title, type: String, desc: 'Title of a wiki page' + requires :content, type: String, desc: 'Content of a wiki page' + use :common_wiki_page_params end post ':id/wikis' do authorize! :create_wiki, user_project @@ -72,7 +72,10 @@ module API success Entities::WikiPage end params do - use :wiki_page_params + optional :title, type: String, desc: 'Title of a wiki page' + optional :content, type: String, desc: 'Content of a wiki page' + use :common_wiki_page_params + at_least_one_of :content, :title, :format end put ':id/wikis/:slug' do authorize! :create_wiki, user_project diff --git a/lib/banzai/filter/footnote_filter.rb b/lib/banzai/filter/footnote_filter.rb index 97527976437..de133774dfa 100644 --- a/lib/banzai/filter/footnote_filter.rb +++ b/lib/banzai/filter/footnote_filter.rb @@ -29,21 +29,30 @@ module Banzai # Sanitization stripped off the section wrapper - add it back in first_footnote.parent.wrap('<section class="footnotes">') rand_suffix = "-#{random_number}" + modified_footnotes = {} doc.css('sup > a[id]').each do |link_node| ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX) footnote_node = doc.at_css("li[id=#{fn_id(ref_num)}]") - backref_node = footnote_node.at_css("a[href=\"##{fnref_id(ref_num)}\"]") - if ref_num =~ INTEGER_PATTERN && footnote_node && backref_node - link_node[:href] += rand_suffix - link_node[:id] += rand_suffix - footnote_node[:id] += rand_suffix - backref_node[:href] += rand_suffix + if INTEGER_PATTERN.match?(ref_num) && (footnote_node || modified_footnotes[ref_num]) + link_node[:href] += rand_suffix + link_node[:id] += rand_suffix # Sanitization stripped off class - add it back in link_node.parent.append_class('footnote-ref') - backref_node.append_class('footnote-backref') + + unless modified_footnotes[ref_num] + footnote_node[:id] += rand_suffix + backref_node = footnote_node.at_css("a[href=\"##{fnref_id(ref_num)}\"]") + + if backref_node + backref_node[:href] += rand_suffix + backref_node.append_class('footnote-backref') + end + + modified_footnotes[ref_num] = true + end end end diff --git a/lib/feature.rb b/lib/feature.rb index e59cd70f822..749c861d740 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -111,11 +111,11 @@ class Feature end def gate_specified? - %i(user project feature_group).any? { |key| params.key?(key) } + %i(user project group feature_group).any? { |key| params.key?(key) } end def targets - [feature_group, user, project].compact + [feature_group, user, project, group].compact end private @@ -139,5 +139,11 @@ class Feature Project.find_by_full_path(params[:project]) end + + def group + return unless params.key?(:group) + + Group.find_by_full_path(params[:group]) + end end end diff --git a/lib/gitlab/ci/build/policy/changes.rb b/lib/gitlab/ci/build/policy/changes.rb index 1663c875426..9c705a1cd3e 100644 --- a/lib/gitlab/ci/build/policy/changes.rb +++ b/lib/gitlab/ci/build/policy/changes.rb @@ -10,7 +10,7 @@ module Gitlab end def satisfied_by?(pipeline, seed) - return true unless pipeline.branch_updated? + return true if pipeline.modified_paths.nil? pipeline.modified_paths.any? do |path| @globs.any? do |glob| diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index e369d26f22f..527a432d72d 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -54,6 +54,8 @@ variables: DOCKER_DRIVER: overlay2 + ROLLOUT_RESOURCE_TYPE: deployment + stages: - build - test @@ -740,7 +742,7 @@ rollout 100%: chart/ fi - kubectl rollout status -n "$KUBE_NAMESPACE" -w "deployment/$name" + kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name" } function scale() { diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb new file mode 100644 index 00000000000..0f3f5cb3c08 --- /dev/null +++ b/lib/gitlab/danger/helper.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true +require 'net/http' +require 'json' + +require_relative 'teammate' + +module Gitlab + module Danger + module Helper + ROULETTE_DATA_URL = URI.parse('https://about.gitlab.com/roulette.json').freeze + + # Returns a list of all files that have been added, modified or renamed. + # `git.modified_files` might contain paths that already have been renamed, + # so we need to remove them from the list. + # + # Considering these changes: + # + # - A new_file.rb + # - D deleted_file.rb + # - M modified_file.rb + # - R renamed_file_before.rb -> renamed_file_after.rb + # + # it will return + # ``` + # [ 'new_file.rb', 'modified_file.rb', 'renamed_file_after.rb' ] + # ``` + # + # @return [Array<String>] + def all_changed_files + Set.new + .merge(git.added_files.to_a) + .merge(git.modified_files.to_a) + .merge(git.renamed_files.map { |x| x[:after] }) + .subtract(git.renamed_files.map { |x| x[:before] }) + .to_a + .sort + end + + def ee? + ENV['CI_PROJECT_NAME'] == 'gitlab-ee' || File.exist?('../../CHANGELOG-EE.md') + end + + def project_name + ee? ? 'gitlab-ee' : 'gitlab-ce' + end + + # Looks up the current list of GitLab team members and parses it into a + # useful form + # + # @return [Array<Teammate>] + def team + @team ||= + begin + rsp = Net::HTTP.get_response(ROULETTE_DATA_URL) + raise "Failed to read #{ROULETTE_DATA_URL}: #{rsp.code} #{rsp.message}" unless + rsp.is_a?(Net::HTTPSuccess) + + data = JSON.parse(rsp.body) + data.map { |hash| ::Gitlab::Danger::Teammate.new(hash) } + rescue JSON::ParserError + raise "Failed to parse JSON response from #{ROULETTE_DATA_URL}" + end + end + + # Like +team+, but only returns teammates in the current project, based on + # project_name. + # + # @return [Array<Teammate>] + def project_team + team.select { |member| member.in_project?(project_name) } + end + + # @return [Hash<String,Array<String>>] + def changes_by_category + all_changed_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, hash| + hash[category_for_file(file)] << file + end + end + + # Determines the category a file is in, e.g., `:frontend` or `:backend` + # @return[Symbol] + def category_for_file(file) + _, category = CATEGORIES.find { |regexp, _| regexp.match?(file) } + + category || :unknown + end + + # Returns the GFM for a category label, making its best guess if it's not + # a category we know about. + # + # @return[String] + def label_for_category(category) + CATEGORY_LABELS.fetch(category, "~#{category}") + end + + CATEGORY_LABELS = { + docs: "~Documentation", + none: "", + qa: "~QA" + }.freeze + + # rubocop:disable Style/RegexpLiteral + CATEGORIES = { + %r{\Adoc/} => :docs, + %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs, + + %r{\A(ee/)?app/(assets|views)/} => :frontend, + %r{\A(ee/)?public/} => :frontend, + %r{\A(ee/)?spec/javascripts/} => :frontend, + %r{\A(ee/)?vendor/assets/} => :frontend, + %r{\A(jest\.config\.js|package\.json|yarn\.lock)\z} => :frontend, + + %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, + %r{\A(ee/)?(bin|config|danger|generator_templates|lib|rubocop|scripts)/} => :backend, + %r{\A(ee/)?spec/(?!javascripts)[^/]+} => :backend, + %r{\A(ee/)?vendor/(?!assets)[^/]+} => :backend, + %r{\A(ee/)?vendor/(languages\.yml|licenses\.csv)\z} => :backend, + %r{\A(Dangerfile|Gemfile|Gemfile.lock|Procfile|Rakefile|\.gitlab-ci\.yml)\z} => :backend, + %r{\A[A-Z_]+_VERSION\z} => :backend, + + %r{\A(ee/)?db/} => :database, + %r{\A(ee/)?qa/} => :qa, + + # Files that don't fit into any category are marked with :none + %r{\A(ee/)?changelogs/} => :none, + + # Fallbacks in case the above patterns miss anything + %r{\.rb\z} => :backend, + %r{\.(md|txt)\z} => :docs, + %r{\.js\z} => :frontend + }.freeze + # rubocop:enable Style/RegexpLiteral + end + end +end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb new file mode 100644 index 00000000000..4b822aa86c5 --- /dev/null +++ b/lib/gitlab/danger/teammate.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Danger + class Teammate + attr_reader :name, :username, :projects + + def initialize(options = {}) + @name = options['name'] + @username = options['username'] + @projects = options['projects'] + end + + def markdown_name + "[#{name}](https://gitlab.com/#{username}) (`@#{username}`)" + end + + def in_project?(name) + projects&.has_key?(name) + end + + # Traintainers also count as reviewers + def reviewer?(project, category) + capabilities(project) == "reviewer #{category}" || traintainer?(project, category) + end + + def traintainer?(project, category) + capabilities(project) == "trainee_maintainer #{category}" + end + + def maintainer?(project, category) + capabilities(project) == "maintainer #{category}" + end + + private + + def capabilities(project) + projects.fetch(project, '') + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index e410d5a8333..c9d89d56884 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -293,6 +293,10 @@ module Gitlab end end + def viewer + rich_viewer || simple_viewer + end + def simple_viewer @simple_viewer ||= simple_viewer_class.new(self) end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 08e30214b46..0891f79198d 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -52,6 +52,14 @@ module Gitlab Gitlab::EtagCaching::Router::Route.new( %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z), 'environments' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/import/github/realtime_changes\.json\z), + 'realtime_changes_import_github' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/import/gitea/realtime_changes\.json\z), + 'realtime_changes_import_gitea' ) ].freeze diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 54bbd531398..593a3676519 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -491,6 +491,13 @@ module Gitlab end end + # Return total diverging commits count + def diverging_commit_count(from, to, max_count:) + wrapped_gitaly_errors do + gitaly_commit_client.diverging_commit_count(from, to, max_count: max_count) + end + end + # Mimic the `git clean` command and recursively delete untracked files. # Valid keys that can be passed in the +options+ hash are: # diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 4e46cb9f05c..ea12424eb4a 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -150,6 +150,17 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end + def diverging_commit_count(from, to, max_count:) + request = Gitaly::CountDivergingCommitsRequest.new( + repository: @gitaly_repo, + from: encode_binary(from), + to: encode_binary(to), + max_count: max_count + ) + response = GitalyClient.call(@repository.storage, :commit_service, :count_diverging_commits, request, timeout: GitalyClient.medium_timeout) + [response.left_count, response.right_count] + end + def list_last_commits_for_tree(revision, path, offset: 0, limit: 25) request = Gitaly::ListLastCommitsForTreeRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index ae7c4cf1b38..e294173f992 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -67,6 +67,36 @@ module Gitlab def insert_git_data(merge_request, already_exists) insert_or_replace_git_data(merge_request, pull_request.source_branch_sha, pull_request.target_branch_sha, already_exists) + # We need to create the branch after the merge request is + # populated to ensure the merge request is in the right state + # when the branch is created. + create_source_branch_if_not_exists(merge_request) + end + + # An imported merge request will not be mergeable unless the + # source branch exists. For pull requests from forks, the source + # branch will be in the form of + # "github/fork/{project-name}/{source_branch}". This branch will never + # exist, so we create it here. + # + # Note that we only create the branch if the merge request is still open. + # For projects that have many pull requests, we assume that if it's closed + # the branch has already been deleted. + def create_source_branch_if_not_exists(merge_request) + return unless merge_request.open? + + source_branch = pull_request.formatted_source_branch + + return if project.repository.branch_exists?(source_branch) + + project.repository.add_branch(merge_request.author, source_branch, pull_request.source_branch_sha) + rescue Gitlab::Git::CommandError => e + Gitlab::Sentry.track_acceptable_exception(e, + extra: { + source_branch: source_branch, + project_id: merge_request.project.id, + merge_request_id: merge_request.id + }) end end end diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index 593b491a837..0ccc4bfaed3 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -76,10 +76,10 @@ module Gitlab # Returns a formatted source branch. # # For cross-project pull requests the branch name will be in the format - # `owner-name:branch-name`. + # `github/fork/owner-name/branch-name`. def formatted_source_branch if cross_project? && source_repository_owner - "#{source_repository_owner}:#{source_branch}" + "github/fork/#{source_repository_owner}/#{source_branch}" elsif source_branch == target_branch # Sometimes the source and target branch are the same, but GitLab # doesn't support this. This can happen when both the user and diff --git a/lib/gitlab/graphql/authorize/instrumentation.rb b/lib/gitlab/graphql/authorize/instrumentation.rb index d638d2b43ee..2a3d790d67b 100644 --- a/lib/gitlab/graphql/authorize/instrumentation.rb +++ b/lib/gitlab/graphql/authorize/instrumentation.rb @@ -35,10 +35,22 @@ module Gitlab private def build_checker(current_user, abilities) - proc do |obj| + lambda do |value| # Load the elements if they weren't loaded by BatchLoader yet - obj = obj.sync if obj.respond_to?(:sync) - obj if abilities.all? { |ability| Ability.allowed?(current_user, ability, obj) } + value = value.sync if value.respond_to?(:sync) + + check = lambda do |object| + abilities.all? do |ability| + Ability.allowed?(current_user, ability, object) + end + end + + case value + when Array + value.select(&check) + else + value if check.call(value) + end end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 099677a791c..7f8c6d56627 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -133,7 +133,6 @@ excluded_attributes: - :external_diff - :stored_externally - :external_diff_store - - :st_diffs merge_request_diff_files: - :diff - :external_diff_offset diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 947caaaefee..725c1101d70 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -61,7 +61,7 @@ module Gitlab def log_base_data { importer: 'Import/Export', - import_jid: @project&.import_state&.import_jid, + import_jid: @project&.import_state&.jid, project_id: @project&.id, project_path: @project&.full_path } diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 26b81847d37..31e6fc9d8c7 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -30,8 +30,8 @@ module Gitlab end end - def token(expire_time: DEFAULT_EXPIRE_TIME) - HMACToken.new(actor).token(expire_time) + def token + HMACToken.new(actor).token(DEFAULT_EXPIRE_TIME) end def token_valid?(token_to_check) @@ -47,6 +47,15 @@ module Gitlab user? ? :lfs_token : :lfs_deploy_token end + def authentication_payload(repository_http_path) + { + username: actor_name, + lfs_token: token, + repository_http_path: repository_http_path, + expires_in: DEFAULT_EXPIRE_TIME + } + end + private # rubocop:disable Lint/UselessAccessModifier class HMACToken diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 651e241362c..ff3fffe7b95 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -19,7 +19,7 @@ module Gitlab # Returns the name of the series to use for storing method calls. def self.series - @series ||= "#{Metrics.series_prefix}method_calls" + @series ||= "#{::Gitlab::Metrics.series_prefix}method_calls" end # Instruments a class method. @@ -118,7 +118,7 @@ module Gitlab # mod - The module containing the method. # name - The name of the method to instrument. def self.instrument(type, mod, name) - return unless Metrics.enabled? + return unless ::Gitlab::Metrics.enabled? name = name.to_sym target = type == :instance ? mod : mod.singleton_class diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index 85438011cb9..d0c63a862c2 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -65,7 +65,7 @@ module Gitlab # Returns true if the total runtime of this method exceeds the method call # threshold. def above_threshold? - real_time.in_milliseconds >= Metrics.method_call_threshold + real_time.in_milliseconds >= ::Gitlab::Metrics.method_call_threshold end end end diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index 447d03bfca4..cee601ff14c 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -58,11 +58,11 @@ module Gitlab def build_metric!(type, name, options) case type when :gauge - Gitlab::Metrics.gauge(name, options.docstring, options.base_labels, options.multiprocess_mode) + ::Gitlab::Metrics.gauge(name, options.docstring, options.base_labels, options.multiprocess_mode) when :counter - Gitlab::Metrics.counter(name, options.docstring, options.base_labels) + ::Gitlab::Metrics.counter(name, options.docstring, options.base_labels) when :histogram - Gitlab::Metrics.histogram(name, options.docstring, options.base_labels, options.buckets) + ::Gitlab::Metrics.histogram(name, options.docstring, options.base_labels, options.buckets) when :summary raise NotImplementedError, "summary metrics are not currently supported" else diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 74c956ab5af..26aa0910047 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -8,15 +8,15 @@ module Gitlab end def self.http_request_total - @http_request_total ||= Gitlab::Metrics.counter(:http_requests_total, 'Request count') + @http_request_total ||= ::Gitlab::Metrics.counter(:http_requests_total, 'Request count') end def self.rack_uncaught_errors_count - @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') + @rack_uncaught_errors_count ||= ::Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') end def self.http_request_duration_seconds - @http_request_duration_seconds ||= Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', + @http_request_duration_seconds ||= ::Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) end diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index c4c38b23a55..5138b37f83e 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -10,7 +10,7 @@ module Gitlab # statistics, etc. class InfluxSampler < BaseSampler # interval - The sampling interval in seconds. - def initialize(interval = Metrics.settings[:sample_interval]) + def initialize(interval = ::Gitlab::Metrics.settings[:sample_interval]) super(interval) @last_step = nil @@ -32,7 +32,7 @@ module Gitlab end def flush - Metrics.submit_metrics(@metrics.map(&:to_hash)) + ::Gitlab::Metrics.submit_metrics(@metrics.map(&:to_hash)) end def sample_memory_usage diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 232a58a7d69..18a69321905 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -24,14 +24,14 @@ module Gitlab def init_metrics metrics = {} - metrics[:sampler_duration] = Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) - metrics[:total_time] = Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + metrics[:sampler_duration] = ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) + metrics[:total_time] = ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) GC.stat.keys.each do |key| - metrics[key] = Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) + metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) - metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) + metrics[:memory_usage] = ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) + metrics[:file_descriptors] = ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) metrics end diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index 4c5b849cc51..bec64e864b3 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -9,11 +9,11 @@ module Gitlab end def unicorn_active_connections - @unicorn_active_connections ||= Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) + @unicorn_active_connections ||= ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) end def unicorn_queued_connections - @unicorn_queued_connections ||= Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) + @unicorn_queued_connections ||= ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) end def enabled? diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb index 56e106b9612..71a5406815f 100644 --- a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb +++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb @@ -9,7 +9,7 @@ module Gitlab LOG_FILENAME = File.join(Rails.root, 'log', 'sidekiq_exporter.log') def enabled? - Gitlab::Metrics.metrics_folder_present? && settings.enabled + ::Gitlab::Metrics.metrics_folder_present? && settings.enabled end def settings diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index f633e1a9d7c..01db507761b 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -64,7 +64,7 @@ module Gitlab end def metric_cache_operation_duration_seconds - @metric_cache_operation_duration_seconds ||= Gitlab::Metrics.histogram( + @metric_cache_operation_duration_seconds ||= ::Gitlab::Metrics.histogram( :gitlab_cache_operation_duration_seconds, 'Cache access time', Transaction::BASE_LABELS.merge({ action: nil }), @@ -73,7 +73,7 @@ module Gitlab end def metric_cache_misses_total - @metric_cache_misses_total ||= Gitlab::Metrics.counter( + @metric_cache_misses_total ||= ::Gitlab::Metrics.counter( :gitlab_cache_misses_total, 'Cache read miss', Transaction::BASE_LABELS diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 468d7cb56fc..e91803ecd62 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -64,7 +64,7 @@ module Gitlab end def add_metric(series, values, tags = {}) - @metrics << Metric.new("#{Metrics.series_prefix}#{series}", values, tags) + @metrics << Metric.new("#{::Gitlab::Metrics.series_prefix}#{series}", values, tags) end # Tracks a business level event @@ -127,7 +127,7 @@ module Gitlab hash end - Metrics.submit_metrics(submit_hashes) + ::Gitlab::Metrics.submit_metrics(submit_hashes) end def labels diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index 96c6a0a7d28..a147e165262 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -7,6 +7,8 @@ module Gitlab module Middleware class RailsQueueDuration + GITLAB_RAILS_QUEUE_DURATION_KEY = 'GITLAB_RAILS_QUEUE_DURATION' + def initialize(app) @app = app end @@ -19,6 +21,7 @@ module Gitlab duration = Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000 trans.set(:rails_queue_duration, duration) metric_rails_queue_duration_seconds.observe(trans.labels, duration / 1_000) + env[GITLAB_RAILS_QUEUE_DURATION_KEY] = duration.round(2) end @app.call(env) diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb index ec1f00a3a91..e45ac5d4765 100644 --- a/lib/gitlab/sql/recursive_cte.rb +++ b/lib/gitlab/sql/recursive_cte.rb @@ -48,7 +48,7 @@ module Gitlab # # alias_table - The Arel table to use as the alias. def alias_to(alias_table) - Arel::Nodes::As.new(table, alias_table) + Arel::Nodes::As.new(table, Arel::Table.new(alias_table.name.tr('.', '_'))) end # Applies the CTE to the given relation, returning a new one that will diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 6bfcf83f388..0101ccc046a 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -64,12 +64,12 @@ module Gitlab group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type), clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), - clusters_applications_helm: count(::Clusters::Applications::Helm.installed), - clusters_applications_ingress: count(::Clusters::Applications::Ingress.installed), - clusters_applications_cert_managers: count(::Clusters::Applications::CertManager.installed), - clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.installed), - clusters_applications_runner: count(::Clusters::Applications::Runner.installed), - clusters_applications_knative: count(::Clusters::Applications::Knative.installed), + clusters_applications_helm: count(::Clusters::Applications::Helm.available), + clusters_applications_ingress: count(::Clusters::Applications::Ingress.available), + clusters_applications_cert_managers: count(::Clusters::Applications::CertManager.available), + clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.available), + clusters_applications_runner: count(::Clusters::Applications::Runner.available), + clusters_applications_knative: count(::Clusters::Applications::Knative.available), in_review_folder: count(::Environment.in_review_folder), groups: count(Group), issues: count(Issue), @@ -90,8 +90,14 @@ module Gitlab todos: count(Todo), uploads: count(Upload), web_hooks: count(WebHook) - }.merge(services_usage).merge(approximate_counts) - } + } + .merge(services_usage) + .merge(approximate_counts) + }.tap do |data| + if Feature.enabled?(:group_overview_security_dashboard) + data[:counts][:user_preferences] = user_preferences_usage + end + end end # rubocop: enable CodeReuse/ActiveRecord @@ -159,6 +165,10 @@ module Gitlab } end + def user_preferences_usage + {} # augmented in EE + end + def count(relation, fallback: -1) relation.count rescue ActiveRecord::StatementInvalid diff --git a/lib/gitlab/wiki_file_finder.rb b/lib/gitlab/wiki_file_finder.rb index 5303b3582ab..e9be6db50da 100644 --- a/lib/gitlab/wiki_file_finder.rb +++ b/lib/gitlab/wiki_file_finder.rb @@ -2,8 +2,6 @@ module Gitlab class WikiFileFinder < FileFinder - BATCH_SIZE = 100 - attr_reader :repository def initialize(project, ref) @@ -19,7 +17,7 @@ module Gitlab safe_query = Regexp.new(safe_query, Regexp::IGNORECASE) filenames = repository.ls_files(ref) - filenames.grep(safe_query).first(BATCH_SIZE) + filenames.grep(safe_query) end end end diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index 4beb94eeb8e..b1db4dc94a6 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -10,6 +10,7 @@ namespace :dev do desc "GitLab | Eager load application" task load: :environment do + Rails.configuration.eager_load = true Rails.application.eager_load! end end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index f71e69987cb..e876b23d43f 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -25,6 +25,11 @@ namespace :gitlab do puts "" end + # In production, we might want to prevent ourselves from shooting + # ourselves in the foot, so let's only do this in a test or + # development environment. + terminate_all_connections unless Rails.env.production? + Rake::Task["db:reset"].invoke Rake::Task["add_limits_mysql"].invoke Rake::Task["setup_postgresql"].invoke @@ -33,4 +38,24 @@ namespace :gitlab do puts "Quitting...".color(:red) exit 1 end + + # If there are any clients connected to the DB, PostgreSQL won't let + # you drop the database. It's possible that Sidekiq, Unicorn, or + # some other client will be hanging onto a connection, preventing + # the DROP DATABASE from working. To workaround this problem, this + # method terminates all the connections so that a subsequent DROP + # will work. + def self.terminate_all_connections + return false unless Gitlab::Database.postgresql? + + cmd = <<~SQL + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE datname = current_database() + AND pid <> pg_backend_pid(); + SQL + + ActiveRecord::Base.connection.execute(cmd)&.result_status == PG::PGRES_TUPLES_OK + rescue ActiveRecord::NoDatabaseError + end end |