diff options
Diffstat (limited to 'lib')
44 files changed, 481 insertions, 199 deletions
diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 9d23daafe95..8defc59224d 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -323,6 +323,22 @@ module API present paginate(commit.merge_requests), with: Entities::MergeRequestBasic end + + desc "Get a commit's GPG signature" do + success Entities::CommitSignature + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end + get ':id/repository/commits/:sha/signature', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do + commit = user_project.commit(params[:sha]) + not_found! 'Commit' unless commit + + signature = commit.signature + not_found! 'GPG Signature' unless signature + + present signature, with: Entities::CommitSignature + end end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index beb8ce349b4..27da2c2e5ed 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -369,8 +369,9 @@ module API end class Commit < Grape::Entity - expose :id, :short_id, :title, :created_at + expose :id, :short_id, :created_at expose :parent_ids + expose :full_title, as: :title expose :safe_message, as: :message expose :author_name, :author_email, :authored_date expose :committer_name, :committer_email, :committed_date @@ -391,6 +392,13 @@ module API expose :project_id end + class CommitSignature < Grape::Entity + expose :gpg_key_id + expose :gpg_key_primary_keyid, :gpg_key_user_name, :gpg_key_user_email + expose :verification_status + expose :gpg_key_subkey_id + end + class BasicRef < Grape::Entity expose :type, :name end @@ -731,6 +739,12 @@ module API def build_available?(options) options[:project]&.feature_available?(:builds, options[:current_user]) end + + expose :user do + expose :can_merge do |merge_request, options| + merge_request.can_be_merged_by?(options[:current_user]) + end + end end class MergeRequestChanges < MergeRequest @@ -1003,7 +1017,7 @@ module API end class LabelBasic < Grape::Entity - expose :id, :name, :color, :description + expose :id, :name, :color, :description, :text_color end class Label < LabelBasic @@ -1031,6 +1045,9 @@ module API expose :priority do |label, options| label.priority(options[:parent]) end + expose :is_project_label do |label, options| + label.is_a?(::ProjectLabel) + end end class List < Grape::Entity diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 4eaaca96b49..fe78049af87 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -81,6 +81,14 @@ module API Gitlab::GlRepository.gl_repository(project, wiki?) end + def gl_project_path + if wiki? + project.wiki.full_path + else + project.full_path + end + end + # Return the repository depending on whether we want the wiki or the # regular repository def repository diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 9488b3469d9..70b32f7d758 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -77,6 +77,7 @@ module API when ::Gitlab::GitAccessResult::Success payload = { gl_repository: gl_repository, + gl_project_path: gl_project_path, gl_id: Gitlab::GlId.gl_id(user), gl_username: user&.username, git_config_options: [], @@ -117,13 +118,7 @@ module API raise ActiveRecord::RecordNotFound.new("No key_id or user_id passed!") end - token_handler = Gitlab::LfsToken.new(actor) - - { - username: token_handler.actor_name, - lfs_token: token_handler.token, - repository_http_path: project.http_url_to_repo - } + Gitlab::LfsToken.new(actor).authentication_payload(project.http_url_to_repo) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/services.rb b/lib/api/services.rb index 637b5a8a89a..163c7505a65 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -368,46 +368,9 @@ module API name: :webhook, type: String, desc: 'The Hangouts Chat webhook. e.g. https://chat.googleapis.com/v1/spaces…' - } - ], - 'hipchat' => [ - { - required: true, - name: :token, - type: String, - desc: 'The room token' - }, - { - required: false, - name: :room, - type: String, - desc: 'The room name or ID' - }, - { - required: false, - name: :color, - type: String, - desc: 'The room color' }, - { - required: false, - name: :notify, - type: Boolean, - desc: 'Enable notifications' - }, - { - required: false, - name: :api_version, - type: String, - desc: 'Leave blank for default (v2)' - }, - { - required: false, - name: :server, - type: String, - desc: 'Leave blank for default. https://hipchat.example.com' - } - ], + CHAT_NOTIFICATION_EVENTS + ].flatten, 'irker' => [ { required: true, @@ -691,7 +654,6 @@ module API ExternalWikiService, FlowdockService, HangoutsChatService, - HipchatService, IrkerService, JiraService, KubernetesService, diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index ef0e3decc2c..994074ddc67 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -11,9 +11,7 @@ module API } end - params :wiki_page_params do - requires :content, type: String, desc: 'Content of a wiki page' - requires :title, type: String, desc: 'Title of a wiki page' + params :common_wiki_page_params do optional :format, type: String, values: ProjectWiki::MARKUPS.values.map(&:to_s), @@ -54,7 +52,9 @@ module API success Entities::WikiPage end params do - use :wiki_page_params + requires :title, type: String, desc: 'Title of a wiki page' + requires :content, type: String, desc: 'Content of a wiki page' + use :common_wiki_page_params end post ':id/wikis' do authorize! :create_wiki, user_project @@ -72,7 +72,10 @@ module API success Entities::WikiPage end params do - use :wiki_page_params + optional :title, type: String, desc: 'Title of a wiki page' + optional :content, type: String, desc: 'Content of a wiki page' + use :common_wiki_page_params + at_least_one_of :content, :title, :format end put ':id/wikis/:slug' do authorize! :create_wiki, user_project diff --git a/lib/banzai/filter/footnote_filter.rb b/lib/banzai/filter/footnote_filter.rb index 97527976437..de133774dfa 100644 --- a/lib/banzai/filter/footnote_filter.rb +++ b/lib/banzai/filter/footnote_filter.rb @@ -29,21 +29,30 @@ module Banzai # Sanitization stripped off the section wrapper - add it back in first_footnote.parent.wrap('<section class="footnotes">') rand_suffix = "-#{random_number}" + modified_footnotes = {} doc.css('sup > a[id]').each do |link_node| ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX) footnote_node = doc.at_css("li[id=#{fn_id(ref_num)}]") - backref_node = footnote_node.at_css("a[href=\"##{fnref_id(ref_num)}\"]") - if ref_num =~ INTEGER_PATTERN && footnote_node && backref_node - link_node[:href] += rand_suffix - link_node[:id] += rand_suffix - footnote_node[:id] += rand_suffix - backref_node[:href] += rand_suffix + if INTEGER_PATTERN.match?(ref_num) && (footnote_node || modified_footnotes[ref_num]) + link_node[:href] += rand_suffix + link_node[:id] += rand_suffix # Sanitization stripped off class - add it back in link_node.parent.append_class('footnote-ref') - backref_node.append_class('footnote-backref') + + unless modified_footnotes[ref_num] + footnote_node[:id] += rand_suffix + backref_node = footnote_node.at_css("a[href=\"##{fnref_id(ref_num)}\"]") + + if backref_node + backref_node[:href] += rand_suffix + backref_node.append_class('footnote-backref') + end + + modified_footnotes[ref_num] = true + end end end diff --git a/lib/gitlab/ci/build/policy/changes.rb b/lib/gitlab/ci/build/policy/changes.rb index 1663c875426..9c705a1cd3e 100644 --- a/lib/gitlab/ci/build/policy/changes.rb +++ b/lib/gitlab/ci/build/policy/changes.rb @@ -10,7 +10,7 @@ module Gitlab end def satisfied_by?(pipeline, seed) - return true unless pipeline.branch_updated? + return true if pipeline.modified_paths.nil? pipeline.modified_paths.any? do |path| @globs.any? do |glob| diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index fc3d4ecdbba..25a32ba0f74 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -57,6 +57,7 @@ test_job: script: - '& "$env:NUNIT_PATH" ".\$env:TEST_FOLDER\Tests.dll"' # running NUnit tests artifacts: + when: always # save test results even when the task fails expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on paths: - '.\TestResult.xml' # saving NUnit results to copy to deploy folder diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb new file mode 100644 index 00000000000..0f3f5cb3c08 --- /dev/null +++ b/lib/gitlab/danger/helper.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true +require 'net/http' +require 'json' + +require_relative 'teammate' + +module Gitlab + module Danger + module Helper + ROULETTE_DATA_URL = URI.parse('https://about.gitlab.com/roulette.json').freeze + + # Returns a list of all files that have been added, modified or renamed. + # `git.modified_files` might contain paths that already have been renamed, + # so we need to remove them from the list. + # + # Considering these changes: + # + # - A new_file.rb + # - D deleted_file.rb + # - M modified_file.rb + # - R renamed_file_before.rb -> renamed_file_after.rb + # + # it will return + # ``` + # [ 'new_file.rb', 'modified_file.rb', 'renamed_file_after.rb' ] + # ``` + # + # @return [Array<String>] + def all_changed_files + Set.new + .merge(git.added_files.to_a) + .merge(git.modified_files.to_a) + .merge(git.renamed_files.map { |x| x[:after] }) + .subtract(git.renamed_files.map { |x| x[:before] }) + .to_a + .sort + end + + def ee? + ENV['CI_PROJECT_NAME'] == 'gitlab-ee' || File.exist?('../../CHANGELOG-EE.md') + end + + def project_name + ee? ? 'gitlab-ee' : 'gitlab-ce' + end + + # Looks up the current list of GitLab team members and parses it into a + # useful form + # + # @return [Array<Teammate>] + def team + @team ||= + begin + rsp = Net::HTTP.get_response(ROULETTE_DATA_URL) + raise "Failed to read #{ROULETTE_DATA_URL}: #{rsp.code} #{rsp.message}" unless + rsp.is_a?(Net::HTTPSuccess) + + data = JSON.parse(rsp.body) + data.map { |hash| ::Gitlab::Danger::Teammate.new(hash) } + rescue JSON::ParserError + raise "Failed to parse JSON response from #{ROULETTE_DATA_URL}" + end + end + + # Like +team+, but only returns teammates in the current project, based on + # project_name. + # + # @return [Array<Teammate>] + def project_team + team.select { |member| member.in_project?(project_name) } + end + + # @return [Hash<String,Array<String>>] + def changes_by_category + all_changed_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, hash| + hash[category_for_file(file)] << file + end + end + + # Determines the category a file is in, e.g., `:frontend` or `:backend` + # @return[Symbol] + def category_for_file(file) + _, category = CATEGORIES.find { |regexp, _| regexp.match?(file) } + + category || :unknown + end + + # Returns the GFM for a category label, making its best guess if it's not + # a category we know about. + # + # @return[String] + def label_for_category(category) + CATEGORY_LABELS.fetch(category, "~#{category}") + end + + CATEGORY_LABELS = { + docs: "~Documentation", + none: "", + qa: "~QA" + }.freeze + + # rubocop:disable Style/RegexpLiteral + CATEGORIES = { + %r{\Adoc/} => :docs, + %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs, + + %r{\A(ee/)?app/(assets|views)/} => :frontend, + %r{\A(ee/)?public/} => :frontend, + %r{\A(ee/)?spec/javascripts/} => :frontend, + %r{\A(ee/)?vendor/assets/} => :frontend, + %r{\A(jest\.config\.js|package\.json|yarn\.lock)\z} => :frontend, + + %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, + %r{\A(ee/)?(bin|config|danger|generator_templates|lib|rubocop|scripts)/} => :backend, + %r{\A(ee/)?spec/(?!javascripts)[^/]+} => :backend, + %r{\A(ee/)?vendor/(?!assets)[^/]+} => :backend, + %r{\A(ee/)?vendor/(languages\.yml|licenses\.csv)\z} => :backend, + %r{\A(Dangerfile|Gemfile|Gemfile.lock|Procfile|Rakefile|\.gitlab-ci\.yml)\z} => :backend, + %r{\A[A-Z_]+_VERSION\z} => :backend, + + %r{\A(ee/)?db/} => :database, + %r{\A(ee/)?qa/} => :qa, + + # Files that don't fit into any category are marked with :none + %r{\A(ee/)?changelogs/} => :none, + + # Fallbacks in case the above patterns miss anything + %r{\.rb\z} => :backend, + %r{\.(md|txt)\z} => :docs, + %r{\.js\z} => :frontend + }.freeze + # rubocop:enable Style/RegexpLiteral + end + end +end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb new file mode 100644 index 00000000000..4b822aa86c5 --- /dev/null +++ b/lib/gitlab/danger/teammate.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Danger + class Teammate + attr_reader :name, :username, :projects + + def initialize(options = {}) + @name = options['name'] + @username = options['username'] + @projects = options['projects'] + end + + def markdown_name + "[#{name}](https://gitlab.com/#{username}) (`@#{username}`)" + end + + def in_project?(name) + projects&.has_key?(name) + end + + # Traintainers also count as reviewers + def reviewer?(project, category) + capabilities(project) == "reviewer #{category}" || traintainer?(project, category) + end + + def traintainer?(project, category) + capabilities(project) == "trainee_maintainer #{category}" + end + + def maintainer?(project, category) + capabilities(project) == "maintainer #{category}" + end + + private + + def capabilities(project) + projects.fetch(project, '') + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index e410d5a8333..c9d89d56884 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -293,6 +293,10 @@ module Gitlab end end + def viewer + rich_viewer || simple_viewer + end + def simple_viewer @simple_viewer ||= simple_viewer_class.new(self) end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 08e30214b46..0891f79198d 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -52,6 +52,14 @@ module Gitlab Gitlab::EtagCaching::Router::Route.new( %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z), 'environments' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/import/github/realtime_changes\.json\z), + 'realtime_changes_import_github' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/import/gitea/realtime_changes\.json\z), + 'realtime_changes_import_gitea' ) ].freeze diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 54bbd531398..593a3676519 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -491,6 +491,13 @@ module Gitlab end end + # Return total diverging commits count + def diverging_commit_count(from, to, max_count:) + wrapped_gitaly_errors do + gitaly_commit_client.diverging_commit_count(from, to, max_count: max_count) + end + end + # Mimic the `git clean` command and recursively delete untracked files. # Valid keys that can be passed in the +options+ hash are: # diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 4e46cb9f05c..ea12424eb4a 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -150,6 +150,17 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end + def diverging_commit_count(from, to, max_count:) + request = Gitaly::CountDivergingCommitsRequest.new( + repository: @gitaly_repo, + from: encode_binary(from), + to: encode_binary(to), + max_count: max_count + ) + response = GitalyClient.call(@repository.storage, :commit_service, :count_diverging_commits, request, timeout: GitalyClient.medium_timeout) + [response.left_count, response.right_count] + end + def list_last_commits_for_tree(revision, path, offset: 0, limit: 25) request = Gitaly::ListLastCommitsForTreeRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index ae7c4cf1b38..e294173f992 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -67,6 +67,36 @@ module Gitlab def insert_git_data(merge_request, already_exists) insert_or_replace_git_data(merge_request, pull_request.source_branch_sha, pull_request.target_branch_sha, already_exists) + # We need to create the branch after the merge request is + # populated to ensure the merge request is in the right state + # when the branch is created. + create_source_branch_if_not_exists(merge_request) + end + + # An imported merge request will not be mergeable unless the + # source branch exists. For pull requests from forks, the source + # branch will be in the form of + # "github/fork/{project-name}/{source_branch}". This branch will never + # exist, so we create it here. + # + # Note that we only create the branch if the merge request is still open. + # For projects that have many pull requests, we assume that if it's closed + # the branch has already been deleted. + def create_source_branch_if_not_exists(merge_request) + return unless merge_request.open? + + source_branch = pull_request.formatted_source_branch + + return if project.repository.branch_exists?(source_branch) + + project.repository.add_branch(merge_request.author, source_branch, pull_request.source_branch_sha) + rescue Gitlab::Git::CommandError => e + Gitlab::Sentry.track_acceptable_exception(e, + extra: { + source_branch: source_branch, + project_id: merge_request.project.id, + merge_request_id: merge_request.id + }) end end end diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index 593b491a837..0ccc4bfaed3 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -76,10 +76,10 @@ module Gitlab # Returns a formatted source branch. # # For cross-project pull requests the branch name will be in the format - # `owner-name:branch-name`. + # `github/fork/owner-name/branch-name`. def formatted_source_branch if cross_project? && source_repository_owner - "#{source_repository_owner}:#{source_branch}" + "github/fork/#{source_repository_owner}/#{source_branch}" elsif source_branch == target_branch # Sometimes the source and target branch are the same, but GitLab # doesn't support this. This can happen when both the user and diff --git a/lib/gitlab/graphql/authorize/instrumentation.rb b/lib/gitlab/graphql/authorize/instrumentation.rb index d638d2b43ee..2a3d790d67b 100644 --- a/lib/gitlab/graphql/authorize/instrumentation.rb +++ b/lib/gitlab/graphql/authorize/instrumentation.rb @@ -35,10 +35,22 @@ module Gitlab private def build_checker(current_user, abilities) - proc do |obj| + lambda do |value| # Load the elements if they weren't loaded by BatchLoader yet - obj = obj.sync if obj.respond_to?(:sync) - obj if abilities.all? { |ability| Ability.allowed?(current_user, ability, obj) } + value = value.sync if value.respond_to?(:sync) + + check = lambda do |object| + abilities.all? do |ability| + Ability.allowed?(current_user, ability, object) + end + end + + case value + when Array + value.select(&check) + else + value if check.call(value) + end end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 099677a791c..7f8c6d56627 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -133,7 +133,6 @@ excluded_attributes: - :external_diff - :stored_externally - :external_diff_store - - :st_diffs merge_request_diff_files: - :diff - :external_diff_offset diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 947caaaefee..725c1101d70 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -61,7 +61,7 @@ module Gitlab def log_base_data { importer: 'Import/Export', - import_jid: @project&.import_state&.import_jid, + import_jid: @project&.import_state&.jid, project_id: @project&.id, project_path: @project&.full_path } diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index b9903e37f40..7dfd9ed4f35 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -20,14 +20,7 @@ module Gitlab kubeclient.create_pod(command.pod_resource) end - def update(command) - namespace.ensure_exists! - - update_config_map(command) - - delete_pod!(command.pod_name) - kubeclient.create_pod(command.pod_resource) - end + alias_method :update, :install ## # Returns Pod phase @@ -62,6 +55,8 @@ module Gitlab def create_config_map(command) command.config_map_resource.tap do |config_map_resource| + break unless config_map_resource + if config_map_exists?(config_map_resource) kubeclient.update_config_map(config_map_resource) else diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index a1ab5e048ac..f931248b747 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -42,8 +42,17 @@ module Gitlab 'helm repo update' if repository end + # Uses `helm upgrade --install` which means we can use this for both + # installation and uprade of applications def install_command - command = ['helm', 'install', chart] + install_command_flags + command = ['helm', 'upgrade', name, chart] + + install_flag + + reset_values_flag + + optional_tls_flags + + optional_version_flag + + rbac_create_flag + + namespace_flag + + value_flag command.shelljoin end @@ -56,17 +65,20 @@ module Gitlab postinstall.join("\n") if postinstall end - def install_command_flags - name_flag = ['--name', name] - namespace_flag = ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] - value_flag = ['-f', "/data/helm/#{name}/config/values.yaml"] + def install_flag + ['--install'] + end - name_flag + - optional_tls_flags + - optional_version_flag + - rbac_create_flag + - namespace_flag + - value_flag + def reset_values_flag + ['--reset-values'] + end + + def value_flag + ['-f', "/data/helm/#{name}/config/values.yaml"] + end + + def namespace_flag + ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] end def rbac_create_flag diff --git a/lib/gitlab/kubernetes/helm/upgrade_command.rb b/lib/gitlab/kubernetes/helm/upgrade_command.rb deleted file mode 100644 index 9daffc138b5..00000000000 --- a/lib/gitlab/kubernetes/helm/upgrade_command.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class UpgradeCommand - include BaseCommand - include ClientCommand - - attr_reader :name, :chart, :version, :repository, :files - - def initialize(name, chart:, files:, rbac:, version: nil, repository: nil) - @name = name - @chart = chart - @rbac = rbac - @version = version - @files = files - @repository = repository - end - - def generate_script - super + [ - init_command, - wait_for_tiller_command, - repository_command, - script_command - ].compact.join("\n") - end - - def rbac? - @rbac - end - - def pod_name - "upgrade-#{name}" - end - - private - - def script_command - upgrade_flags = "#{optional_version_flag}#{optional_tls_flags}" \ - " --reset-values" \ - " --install" \ - " --namespace #{::Gitlab::Kubernetes::Helm::NAMESPACE}" \ - " -f /data/helm/#{name}/config/values.yaml" - - "helm upgrade #{name} #{chart}#{upgrade_flags}" - end - - def optional_version_flag - " --version #{version}" if version - end - - def optional_tls_flags - return unless files.key?(:'ca.pem') - - " --tls" \ - " --tls-ca-cert #{files_dir}/ca.pem" \ - " --tls-cert #{files_dir}/cert.pem" \ - " --tls-key #{files_dir}/key.pem" - end - end - end - end -end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 26b81847d37..31e6fc9d8c7 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -30,8 +30,8 @@ module Gitlab end end - def token(expire_time: DEFAULT_EXPIRE_TIME) - HMACToken.new(actor).token(expire_time) + def token + HMACToken.new(actor).token(DEFAULT_EXPIRE_TIME) end def token_valid?(token_to_check) @@ -47,6 +47,15 @@ module Gitlab user? ? :lfs_token : :lfs_deploy_token end + def authentication_payload(repository_http_path) + { + username: actor_name, + lfs_token: token, + repository_http_path: repository_http_path, + expires_in: DEFAULT_EXPIRE_TIME + } + end + private # rubocop:disable Lint/UselessAccessModifier class HMACToken diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 651e241362c..ff3fffe7b95 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -19,7 +19,7 @@ module Gitlab # Returns the name of the series to use for storing method calls. def self.series - @series ||= "#{Metrics.series_prefix}method_calls" + @series ||= "#{::Gitlab::Metrics.series_prefix}method_calls" end # Instruments a class method. @@ -118,7 +118,7 @@ module Gitlab # mod - The module containing the method. # name - The name of the method to instrument. def self.instrument(type, mod, name) - return unless Metrics.enabled? + return unless ::Gitlab::Metrics.enabled? name = name.to_sym target = type == :instance ? mod : mod.singleton_class diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index 85438011cb9..d0c63a862c2 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -65,7 +65,7 @@ module Gitlab # Returns true if the total runtime of this method exceeds the method call # threshold. def above_threshold? - real_time.in_milliseconds >= Metrics.method_call_threshold + real_time.in_milliseconds >= ::Gitlab::Metrics.method_call_threshold end end end diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index 447d03bfca4..cee601ff14c 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -58,11 +58,11 @@ module Gitlab def build_metric!(type, name, options) case type when :gauge - Gitlab::Metrics.gauge(name, options.docstring, options.base_labels, options.multiprocess_mode) + ::Gitlab::Metrics.gauge(name, options.docstring, options.base_labels, options.multiprocess_mode) when :counter - Gitlab::Metrics.counter(name, options.docstring, options.base_labels) + ::Gitlab::Metrics.counter(name, options.docstring, options.base_labels) when :histogram - Gitlab::Metrics.histogram(name, options.docstring, options.base_labels, options.buckets) + ::Gitlab::Metrics.histogram(name, options.docstring, options.base_labels, options.buckets) when :summary raise NotImplementedError, "summary metrics are not currently supported" else diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 74c956ab5af..26aa0910047 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -8,15 +8,15 @@ module Gitlab end def self.http_request_total - @http_request_total ||= Gitlab::Metrics.counter(:http_requests_total, 'Request count') + @http_request_total ||= ::Gitlab::Metrics.counter(:http_requests_total, 'Request count') end def self.rack_uncaught_errors_count - @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') + @rack_uncaught_errors_count ||= ::Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') end def self.http_request_duration_seconds - @http_request_duration_seconds ||= Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', + @http_request_duration_seconds ||= ::Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) end diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index c4c38b23a55..5138b37f83e 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -10,7 +10,7 @@ module Gitlab # statistics, etc. class InfluxSampler < BaseSampler # interval - The sampling interval in seconds. - def initialize(interval = Metrics.settings[:sample_interval]) + def initialize(interval = ::Gitlab::Metrics.settings[:sample_interval]) super(interval) @last_step = nil @@ -32,7 +32,7 @@ module Gitlab end def flush - Metrics.submit_metrics(@metrics.map(&:to_hash)) + ::Gitlab::Metrics.submit_metrics(@metrics.map(&:to_hash)) end def sample_memory_usage diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 232a58a7d69..18a69321905 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -24,14 +24,14 @@ module Gitlab def init_metrics metrics = {} - metrics[:sampler_duration] = Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) - metrics[:total_time] = Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + metrics[:sampler_duration] = ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) + metrics[:total_time] = ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) GC.stat.keys.each do |key| - metrics[key] = Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) + metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) - metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) + metrics[:memory_usage] = ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) + metrics[:file_descriptors] = ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) metrics end diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index 4c5b849cc51..bec64e864b3 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -9,11 +9,11 @@ module Gitlab end def unicorn_active_connections - @unicorn_active_connections ||= Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) + @unicorn_active_connections ||= ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) end def unicorn_queued_connections - @unicorn_queued_connections ||= Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) + @unicorn_queued_connections ||= ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) end def enabled? diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb index 56e106b9612..71a5406815f 100644 --- a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb +++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb @@ -9,7 +9,7 @@ module Gitlab LOG_FILENAME = File.join(Rails.root, 'log', 'sidekiq_exporter.log') def enabled? - Gitlab::Metrics.metrics_folder_present? && settings.enabled + ::Gitlab::Metrics.metrics_folder_present? && settings.enabled end def settings diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index f633e1a9d7c..01db507761b 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -64,7 +64,7 @@ module Gitlab end def metric_cache_operation_duration_seconds - @metric_cache_operation_duration_seconds ||= Gitlab::Metrics.histogram( + @metric_cache_operation_duration_seconds ||= ::Gitlab::Metrics.histogram( :gitlab_cache_operation_duration_seconds, 'Cache access time', Transaction::BASE_LABELS.merge({ action: nil }), @@ -73,7 +73,7 @@ module Gitlab end def metric_cache_misses_total - @metric_cache_misses_total ||= Gitlab::Metrics.counter( + @metric_cache_misses_total ||= ::Gitlab::Metrics.counter( :gitlab_cache_misses_total, 'Cache read miss', Transaction::BASE_LABELS diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 468d7cb56fc..e91803ecd62 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -64,7 +64,7 @@ module Gitlab end def add_metric(series, values, tags = {}) - @metrics << Metric.new("#{Metrics.series_prefix}#{series}", values, tags) + @metrics << Metric.new("#{::Gitlab::Metrics.series_prefix}#{series}", values, tags) end # Tracks a business level event @@ -127,7 +127,7 @@ module Gitlab hash end - Metrics.submit_metrics(submit_hashes) + ::Gitlab::Metrics.submit_metrics(submit_hashes) end def labels diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index 96c6a0a7d28..a147e165262 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -7,6 +7,8 @@ module Gitlab module Middleware class RailsQueueDuration + GITLAB_RAILS_QUEUE_DURATION_KEY = 'GITLAB_RAILS_QUEUE_DURATION' + def initialize(app) @app = app end @@ -19,6 +21,7 @@ module Gitlab duration = Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000 trans.set(:rails_queue_duration, duration) metric_rails_queue_duration_seconds.observe(trans.labels, duration / 1_000) + env[GITLAB_RAILS_QUEUE_DURATION_KEY] = duration.round(2) end @app.call(env) diff --git a/lib/gitlab/patch/sprockets_base_file_digest_key.rb b/lib/gitlab/patch/sprockets_base_file_digest_key.rb index 3925cdbbada..1c472638145 100644 --- a/lib/gitlab/patch/sprockets_base_file_digest_key.rb +++ b/lib/gitlab/patch/sprockets_base_file_digest_key.rb @@ -9,7 +9,7 @@ module Gitlab def file_digest(path) if stat = self.stat(path) digest = self.stat_digest(path, stat) - integrity_uri = self.hexdigest_integrity_uri(digest) + integrity_uri = self.integrity_uri(digest) key = Sprockets::UnloadedAsset.new(path, self).file_digest_key(integrity_uri) cache.fetch(key) do diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 3bfd6ee892c..ef656e5b2ce 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -2,14 +2,12 @@ module Gitlab class ProjectTemplate - attr_reader :title, :name, :description, :preview + attr_reader :title, :name, :description, :preview, :logo - def initialize(name, title, description, preview) - @name, @title, @description, @preview = name, title, description, preview + def initialize(name, title, description, preview, logo = 'illustrations/gitlab_logo.svg') + @name, @title, @description, @preview, @logo = name, title, description, preview, logo end - alias_method :logo, :name - def file archive_path.open end @@ -27,9 +25,14 @@ module Gitlab end TEMPLATES_TABLE = [ - ProjectTemplate.new('rails', 'Ruby on Rails', 'Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/rails'), - ProjectTemplate.new('spring', 'Spring', 'Includes an MVC structure, mvnw and pom.xml to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/spring'), - ProjectTemplate.new('express', 'NodeJS Express', 'Includes an MVC structure to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/express') + ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'), + ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'), + ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'), + ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo'), + ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll'), + ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML.'), 'https://gitlab.com/pages/plain-html'), + ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook'), + ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo') ].freeze class << self diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb index ec1f00a3a91..e45ac5d4765 100644 --- a/lib/gitlab/sql/recursive_cte.rb +++ b/lib/gitlab/sql/recursive_cte.rb @@ -48,7 +48,7 @@ module Gitlab # # alias_table - The Arel table to use as the alias. def alias_to(alias_table) - Arel::Nodes::As.new(table, alias_table) + Arel::Nodes::As.new(table, Arel::Table.new(alias_table.name.tr('.', '_'))) end # Applies the CTE to the given relation, returning a new one that will diff --git a/lib/gitlab/tracing.rb b/lib/gitlab/tracing.rb index 3c4db42ac06..0d9b0be1c8e 100644 --- a/lib/gitlab/tracing.rb +++ b/lib/gitlab/tracing.rb @@ -13,5 +13,24 @@ module Gitlab def self.connection_string ENV['GITLAB_TRACING'] end + + def self.tracing_url_template + ENV['GITLAB_TRACING_URL'] + end + + def self.tracing_url_enabled? + enabled? && tracing_url_template.present? + end + + # This will provide a link into the distributed tracing for the current trace, + # if it has been captured. + def self.tracing_url + return unless tracing_url_enabled? + + tracing_url_template % { + correlation_id: Gitlab::CorrelationId.current_id.to_s, + service: Gitlab.process_name + } + end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 6bfcf83f388..0101ccc046a 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -64,12 +64,12 @@ module Gitlab group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type), clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), - clusters_applications_helm: count(::Clusters::Applications::Helm.installed), - clusters_applications_ingress: count(::Clusters::Applications::Ingress.installed), - clusters_applications_cert_managers: count(::Clusters::Applications::CertManager.installed), - clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.installed), - clusters_applications_runner: count(::Clusters::Applications::Runner.installed), - clusters_applications_knative: count(::Clusters::Applications::Knative.installed), + clusters_applications_helm: count(::Clusters::Applications::Helm.available), + clusters_applications_ingress: count(::Clusters::Applications::Ingress.available), + clusters_applications_cert_managers: count(::Clusters::Applications::CertManager.available), + clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.available), + clusters_applications_runner: count(::Clusters::Applications::Runner.available), + clusters_applications_knative: count(::Clusters::Applications::Knative.available), in_review_folder: count(::Environment.in_review_folder), groups: count(Group), issues: count(Issue), @@ -90,8 +90,14 @@ module Gitlab todos: count(Todo), uploads: count(Upload), web_hooks: count(WebHook) - }.merge(services_usage).merge(approximate_counts) - } + } + .merge(services_usage) + .merge(approximate_counts) + }.tap do |data| + if Feature.enabled?(:group_overview_security_dashboard) + data[:counts][:user_preferences] = user_preferences_usage + end + end end # rubocop: enable CodeReuse/ActiveRecord @@ -159,6 +165,10 @@ module Gitlab } end + def user_preferences_usage + {} # augmented in EE + end + def count(relation, fallback: -1) relation.count rescue ActiveRecord::StatementInvalid diff --git a/lib/gitlab/wiki_file_finder.rb b/lib/gitlab/wiki_file_finder.rb index 5303b3582ab..e9be6db50da 100644 --- a/lib/gitlab/wiki_file_finder.rb +++ b/lib/gitlab/wiki_file_finder.rb @@ -2,8 +2,6 @@ module Gitlab class WikiFileFinder < FileFinder - BATCH_SIZE = 100 - attr_reader :repository def initialize(project, ref) @@ -19,7 +17,7 @@ module Gitlab safe_query = Regexp.new(safe_query, Regexp::IGNORECASE) filenames = repository.ls_files(ref) - filenames.grep(safe_query).first(BATCH_SIZE) + filenames.grep(safe_query) end end end diff --git a/lib/peek/views/tracing.rb b/lib/peek/views/tracing.rb new file mode 100644 index 00000000000..0de32a8fdda --- /dev/null +++ b/lib/peek/views/tracing.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Peek + module Views + class Tracing < View + def results + { + tracing_url: Gitlab::Tracing.tracing_url + } + end + end + end +end diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index 4beb94eeb8e..b1db4dc94a6 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -10,6 +10,7 @@ namespace :dev do desc "GitLab | Eager load application" task load: :environment do + Rails.configuration.eager_load = true Rails.application.eager_load! end end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index f71e69987cb..e876b23d43f 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -25,6 +25,11 @@ namespace :gitlab do puts "" end + # In production, we might want to prevent ourselves from shooting + # ourselves in the foot, so let's only do this in a test or + # development environment. + terminate_all_connections unless Rails.env.production? + Rake::Task["db:reset"].invoke Rake::Task["add_limits_mysql"].invoke Rake::Task["setup_postgresql"].invoke @@ -33,4 +38,24 @@ namespace :gitlab do puts "Quitting...".color(:red) exit 1 end + + # If there are any clients connected to the DB, PostgreSQL won't let + # you drop the database. It's possible that Sidekiq, Unicorn, or + # some other client will be hanging onto a connection, preventing + # the DROP DATABASE from working. To workaround this problem, this + # method terminates all the connections so that a subsequent DROP + # will work. + def self.terminate_all_connections + return false unless Gitlab::Database.postgresql? + + cmd = <<~SQL + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE datname = current_database() + AND pid <> pg_backend_pid(); + SQL + + ActiveRecord::Base.connection.execute(cmd)&.result_status == PG::PGRES_TUPLES_OK + rescue ActiveRecord::NoDatabaseError + end end |