diff options
84 files changed, 1288 insertions, 253 deletions
diff --git a/.flayignore b/.flayignore index acac0ce14c9..87cb3507b05 100644 --- a/.flayignore +++ b/.flayignore @@ -6,3 +6,5 @@ app/models/concerns/relative_positioning.rb app/workers/stuck_merge_jobs_worker.rb lib/gitlab/redis/*.rb lib/gitlab/gitaly_client/operation_service.rb +lib/gitlab/background_migration/* +app/models/project_services/kubernetes_service.rb diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4afa953175..9c3556f5cce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -627,6 +627,8 @@ codequality: sast: <<: *except-docs image: registry.gitlab.com/gitlab-org/gl-sast:latest + variables: + CONFIDENCE_LEVEL: 2 before_script: [] script: - /app/bin/run . diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b366ae6f069..ed56da0353d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,6 +50,9 @@ _This notice should stay as the first item in the CONTRIBUTING.md file._ ## Contribute to GitLab +For a first-time step-by-step guide to the contribution process, see +["Contributing to GitLab"](https://about.gitlab.com/contributing/). + Thank you for your interest in contributing to GitLab. This guide details how to contribute to GitLab in a way that is efficient for everyone. diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 4a7076db09a..bd14e8533ef 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -0.77.0 +0.78.0 @@ -349,7 +349,7 @@ group :development, :test do gem 'scss_lint', '~> 0.56.0', require: false gem 'haml_lint', '~> 0.26.0', require: false gem 'simplecov', '~> 0.14.0', require: false - gem 'flay', '~> 2.8.0', require: false + gem 'flay', '~> 2.10.0', require: false gem 'bundler-audit', '~> 0.5.0', require: false gem 'benchmark-ips', '~> 2.3.0', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 2ddf8221a06..cf9f160499d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -211,7 +211,7 @@ GEM fast_gettext (1.4.0) ffaker (2.4.0) ffi (1.9.18) - flay (2.8.1) + flay (2.10.0) erubis (~> 2.7.0) path_expander (~> 1.0) ruby_parser (~> 3.0) @@ -588,7 +588,7 @@ GEM ast (~> 2.3) parslet (1.5.0) blankslate (~> 2.0) - path_expander (1.0.1) + path_expander (1.0.2) peek (1.0.1) concurrent-ruby (>= 0.9.0) concurrent-ruby-ext (>= 0.9.0) @@ -1037,7 +1037,7 @@ DEPENDENCIES faraday (~> 0.12) fast_blank ffaker (~> 2.4) - flay (~> 2.8.0) + flay (~> 2.10.0) flipper (~> 0.11.0) flipper-active_record (~> 0.11.0) flipper-active_support_cache_store (~> 0.11.0) diff --git a/app/assets/javascripts/blob/file_template_mediator.js b/app/assets/javascripts/blob/file_template_mediator.js index 583e5faa506..37074301b51 100644 --- a/app/assets/javascripts/blob/file_template_mediator.js +++ b/app/assets/javascripts/blob/file_template_mediator.js @@ -235,7 +235,7 @@ export default class FileTemplateMediator { } setFilename(name) { - this.$filenameInput.val(name); + this.$filenameInput.val(name).trigger('change'); } getSelected() { diff --git a/app/controllers/admin/services_controller.rb b/app/controllers/admin/services_controller.rb index 4c3d336b3af..a7025b62ad7 100644 --- a/app/controllers/admin/services_controller.rb +++ b/app/controllers/admin/services_controller.rb @@ -1,6 +1,7 @@ class Admin::ServicesController < Admin::ApplicationController include ServiceParams + before_action :whitelist_query_limiting, only: [:index] before_action :service, only: [:edit, :update] def index @@ -37,4 +38,8 @@ class Admin::ServicesController < Admin::ApplicationController def service @service ||= Service.where(id: params[:id], template: true).first end + + def whitelist_query_limiting + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42430') + end end diff --git a/app/controllers/boards/issues_controller.rb b/app/controllers/boards/issues_controller.rb index f8049b20b9f..ee23ee0bcc3 100644 --- a/app/controllers/boards/issues_controller.rb +++ b/app/controllers/boards/issues_controller.rb @@ -2,6 +2,7 @@ module Boards class IssuesController < Boards::ApplicationController include BoardsResponses + before_action :whitelist_query_limiting, only: [:index, :update] before_action :authorize_read_issue, only: [:index] before_action :authorize_create_issue, only: [:create] before_action :authorize_update_issue, only: [:update] @@ -92,5 +93,10 @@ module Boards } ) end + + def whitelist_query_limiting + # Also see https://gitlab.com/gitlab-org/gitlab-ce/issues/42439 + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42428') + end end end diff --git a/app/controllers/import/gitlab_projects_controller.rb b/app/controllers/import/gitlab_projects_controller.rb index 567957ba2cb..f22df992fe9 100644 --- a/app/controllers/import/gitlab_projects_controller.rb +++ b/app/controllers/import/gitlab_projects_controller.rb @@ -1,4 +1,5 @@ class Import::GitlabProjectsController < Import::BaseController + before_action :whitelist_query_limiting, only: [:create] before_action :verify_gitlab_project_import_enabled def new @@ -40,4 +41,8 @@ class Import::GitlabProjectsController < Import::BaseController :path, :namespace_id, :file ) end + + def whitelist_query_limiting + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42437') + end end diff --git a/app/controllers/projects/commits_controller.rb b/app/controllers/projects/commits_controller.rb index 0a40c67368f..1d910e461b1 100644 --- a/app/controllers/projects/commits_controller.rb +++ b/app/controllers/projects/commits_controller.rb @@ -4,6 +4,7 @@ class Projects::CommitsController < Projects::ApplicationController include ExtractsPath include RendersCommits + before_action :whitelist_query_limiting before_action :require_non_empty_project before_action :assign_ref_vars before_action :authorize_download_code! @@ -65,4 +66,8 @@ class Projects::CommitsController < Projects::ApplicationController @commits = @commits.with_pipeline_status @commits = prepare_commits_for_rendering(@commits) end + + def whitelist_query_limiting + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42330') + end end diff --git a/app/controllers/projects/cycle_analytics_controller.rb b/app/controllers/projects/cycle_analytics_controller.rb index 88ac3ad046b..d1b8fd80c4e 100644 --- a/app/controllers/projects/cycle_analytics_controller.rb +++ b/app/controllers/projects/cycle_analytics_controller.rb @@ -3,6 +3,7 @@ class Projects::CycleAnalyticsController < Projects::ApplicationController include ActionView::Helpers::TextHelper include CycleAnalyticsParams + before_action :whitelist_query_limiting, only: [:show] before_action :authorize_read_cycle_analytics! def show @@ -31,4 +32,8 @@ class Projects::CycleAnalyticsController < Projects::ApplicationController permissions: @cycle_analytics.permissions(user: current_user) } end + + def whitelist_query_limiting + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42671') + end end diff --git a/app/controllers/projects/forks_controller.rb b/app/controllers/projects/forks_controller.rb index 68978f8fdd1..f43bba18d81 100644 --- a/app/controllers/projects/forks_controller.rb +++ b/app/controllers/projects/forks_controller.rb @@ -2,6 +2,7 @@ class Projects::ForksController < Projects::ApplicationController include ContinueParams # Authorize + before_action :whitelist_query_limiting, only: [:create] before_action :require_non_empty_project before_action :authorize_download_code! before_action :authenticate_user!, only: [:new, :create] @@ -54,4 +55,8 @@ class Projects::ForksController < Projects::ApplicationController render :error end end + + def whitelist_query_limiting + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42335') + end end diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index 384f18b316c..515cb08f1fc 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -8,6 +8,7 @@ class Projects::IssuesController < Projects::ApplicationController prepend_before_action :authenticate_user!, only: [:new] + before_action :whitelist_query_limiting, only: [:create, :create_merge_request, :move, :bulk_update] before_action :check_issues_available! before_action :issue, except: [:index, :new, :create, :bulk_update] before_action :set_issuables_index, only: [:index] @@ -247,4 +248,13 @@ class Projects::IssuesController < Projects::ApplicationController @finder_type = IssuesFinder super end + + def whitelist_query_limiting + # Also see the following issues: + # + # 1. https://gitlab.com/gitlab-org/gitlab-ce/issues/42423 + # 2. https://gitlab.com/gitlab-org/gitlab-ce/issues/42424 + # 3. https://gitlab.com/gitlab-org/gitlab-ce/issues/42426 + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42422') + end end diff --git a/app/controllers/projects/merge_requests/creations_controller.rb b/app/controllers/projects/merge_requests/creations_controller.rb index 0df80fa700f..a5a2d54ba82 100644 --- a/app/controllers/projects/merge_requests/creations_controller.rb +++ b/app/controllers/projects/merge_requests/creations_controller.rb @@ -4,6 +4,7 @@ class Projects::MergeRequests::CreationsController < Projects::MergeRequests::Ap include RendersCommits skip_before_action :merge_request + before_action :whitelist_query_limiting, only: [:create] before_action :authorize_create_merge_request! before_action :apply_diff_view_cookie!, only: [:diffs, :diff_for_path] before_action :build_merge_request, except: [:create] @@ -125,4 +126,8 @@ class Projects::MergeRequests::CreationsController < Projects::MergeRequests::Ap @project.forked_from_project end end + + def whitelist_query_limiting + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42384') + end end diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb index 2e8a738b6d9..8af4e379f0a 100644 --- a/app/controllers/projects/merge_requests_controller.rb +++ b/app/controllers/projects/merge_requests_controller.rb @@ -7,6 +7,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo include IssuableCollections skip_before_action :merge_request, only: [:index, :bulk_update] + before_action :whitelist_query_limiting, only: [:assign_related_issues, :update] before_action :authorize_update_issuable!, only: [:close, :edit, :update, :remove_wip, :sort] before_action :set_issuables_index, only: [:index] before_action :authenticate_user!, only: [:assign_related_issues] @@ -339,4 +340,9 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo access_denied! unless access_check end + + def whitelist_query_limiting + # Also see https://gitlab.com/gitlab-org/gitlab-ce/issues/42441 + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42438') + end end diff --git a/app/controllers/projects/network_controller.rb b/app/controllers/projects/network_controller.rb index fb68dd771a1..3b10a93e97f 100644 --- a/app/controllers/projects/network_controller.rb +++ b/app/controllers/projects/network_controller.rb @@ -2,6 +2,7 @@ class Projects::NetworkController < Projects::ApplicationController include ExtractsPath include ApplicationHelper + before_action :whitelist_query_limiting before_action :require_non_empty_project before_action :assign_ref_vars before_action :authorize_download_code! @@ -35,4 +36,8 @@ class Projects::NetworkController < Projects::ApplicationController @options[:extended_sha1] = params[:extended_sha1] @commit = @repo.commit(@options[:extended_sha1]) end + + def whitelist_query_limiting + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42333') + end end diff --git a/app/controllers/projects/notes_controller.rb b/app/controllers/projects/notes_controller.rb index 5940fae8dd0..4f8978c93c3 100644 --- a/app/controllers/projects/notes_controller.rb +++ b/app/controllers/projects/notes_controller.rb @@ -2,6 +2,7 @@ class Projects::NotesController < Projects::ApplicationController include NotesActions include ToggleAwardEmoji + before_action :whitelist_query_limiting, only: [:create] before_action :authorize_read_note! before_action :authorize_create_note!, only: [:create] before_action :authorize_resolve_note!, only: [:resolve, :unresolve] @@ -79,4 +80,8 @@ class Projects::NotesController < Projects::ApplicationController access_denied! unless can?(current_user, :create_note, noteable) end + + def whitelist_query_limiting + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42383') + end end diff --git a/app/controllers/projects/pipelines_controller.rb b/app/controllers/projects/pipelines_controller.rb index e146d0d3cd5..78d109cf33e 100644 --- a/app/controllers/projects/pipelines_controller.rb +++ b/app/controllers/projects/pipelines_controller.rb @@ -1,4 +1,5 @@ class Projects::PipelinesController < Projects::ApplicationController + before_action :whitelist_query_limiting, only: [:create, :retry] before_action :pipeline, except: [:index, :new, :create, :charts] before_action :commit, only: [:show, :builds, :failures] before_action :authorize_read_pipeline! @@ -166,4 +167,9 @@ class Projects::PipelinesController < Projects::ApplicationController def commit @commit ||= @pipeline.commit end + + def whitelist_query_limiting + # Also see https://gitlab.com/gitlab-org/gitlab-ce/issues/42343 + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42339') + end end diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 86923909d07..72573e0765d 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -3,6 +3,7 @@ class ProjectsController < Projects::ApplicationController include ExtractsPath include PreviewMarkdown + before_action :whitelist_query_limiting, only: [:create] before_action :authenticate_user!, except: [:index, :show, :activity, :refs] before_action :redirect_git_extension, only: [:show] before_action :project, except: [:index, :new, :create] @@ -405,4 +406,8 @@ class ProjectsController < Projects::ApplicationController # redirect_to request.original_url.sub(%r{\.git/?\Z}, '') if params[:format] == 'git' end + + def whitelist_query_limiting + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440') + end end diff --git a/app/controllers/registrations_controller.rb b/app/controllers/registrations_controller.rb index d9142311b6f..1848c806c41 100644 --- a/app/controllers/registrations_controller.rb +++ b/app/controllers/registrations_controller.rb @@ -1,6 +1,8 @@ class RegistrationsController < Devise::RegistrationsController include Recaptcha::Verify + before_action :whitelist_query_limiting, only: [:destroy] + def new redirect_to(new_user_session_path) end @@ -83,4 +85,8 @@ class RegistrationsController < Devise::RegistrationsController def devise_mapping @devise_mapping ||= Devise.mappings[:user] end + + def whitelist_query_limiting + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42380') + end end diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb index 9024f1df1cd..aa5cf97756f 100644 --- a/app/models/clusters/applications/ingress.rb +++ b/app/models/clusters/applications/ingress.rb @@ -17,8 +17,12 @@ module Clusters 'stable/nginx-ingress' end + def chart_values_file + "#{Rails.root}/vendor/#{name}/values.yaml" + end + def install_command - Gitlab::Kubernetes::Helm::InstallCommand.new(name, chart: chart) + Gitlab::Kubernetes::Helm::InstallCommand.new(name, chart: chart, chart_values_file: chart_values_file) end end end diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb index 99dbd4fbacf..b12c10a84de 100644 --- a/app/models/concerns/storage/legacy_namespace.rb +++ b/app/models/concerns/storage/legacy_namespace.rb @@ -87,20 +87,10 @@ module Storage remove_exports! end - def remove_exports! - Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete)) - end - - def export_path - File.join(Gitlab::ImportExport.storage_path, full_path_was) - end + def remove_legacy_exports! + legacy_export_path = File.join(Gitlab::ImportExport.storage_path, full_path_was) - def full_path_was - if parent - parent.full_path + '/' + path_was - else - path_was - end + FileUtils.rm_rf(legacy_export_path) end end end diff --git a/app/models/group.rb b/app/models/group.rb index 62b1322ebe6..5b7f1b38612 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -274,12 +274,6 @@ class Group < Namespace list_of_ids.reverse.map { |group| variables[group.id] }.compact.flatten end - def full_path_was - return path_was unless has_parent? - - "#{parent.full_path}/#{path_was}" - end - def group_member(user) if group_members.loaded? group_members.find { |gm| gm.user_id == user.id } diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 5010dd73c11..7b82d076975 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -221,6 +221,24 @@ class Namespace < ActiveRecord::Base has_parent? end + def full_path_was + return path_was unless has_parent? + + "#{parent.full_path}/#{path_was}" + end + + # Exports belonging to projects with legacy storage are placed in a common + # subdirectory of the namespace, so a simple `rm -rf` is sufficient to remove + # them. + # + # Exports of projects using hashed storage are placed in a location defined + # only by the project ID, so each must be removed individually. + def remove_exports! + remove_legacy_exports! + + all_projects.with_storage_feature(:repository).find_each(&:remove_exports) + end + private def refresh_access_of_projects_invited_groups diff --git a/app/models/project.rb b/app/models/project.rb index 03c5475c31f..12d5f28f5ea 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -69,6 +69,7 @@ class Project < ActiveRecord::Base before_destroy :remove_private_deploy_keys after_destroy -> { run_after_commit { remove_pages } } + after_destroy :remove_exports after_validation :check_pending_delete @@ -1525,6 +1526,8 @@ class Project < ActiveRecord::Base end def export_path + return nil unless namespace.present? || hashed_storage?(:repository) + File.join(Gitlab::ImportExport.storage_path, disk_path) end @@ -1533,8 +1536,9 @@ class Project < ActiveRecord::Base end def remove_exports - _, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete)) - status.zero? + return nil unless export_path.present? + + FileUtils.rm_rf(export_path) end def full_path_slug diff --git a/app/models/repository.rb b/app/models/repository.rb index edfb236a91a..f1abe5c3e07 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -173,15 +173,7 @@ class Repository end def find_branch(name, fresh_repo: true) - # Since the Repository object may have in-memory index changes, invalidating the memoized Repository object may - # cause unintended side effects. Because finding a branch is a read-only operation, we can safely instantiate - # a new repo here to ensure a consistent state to avoid a libgit2 bug where concurrent access (e.g. via git gc) - # may cause the branch to "disappear" erroneously or have the wrong SHA. - # - # See: https://github.com/libgit2/libgit2/issues/1534 and https://gitlab.com/gitlab-org/gitlab-ce/issues/15392 - raw_repo = fresh_repo ? initialize_raw_repository : raw_repository - - raw_repo.find_branch(name) + raw_repository.find_branch(name, fresh_repo) end def find_tag(name) @@ -721,11 +713,11 @@ class Repository end def branch_names_contains(sha) - refs_contains_sha('branch', sha) + raw_repository.branch_names_contains_sha(sha) end def tag_names_contains(sha) - refs_contains_sha('tag', sha) + raw_repository.tag_names_contains_sha(sha) end def local_branches diff --git a/app/services/system_note_service.rb b/app/services/system_note_service.rb index 06b23cd7076..2253d638e93 100644 --- a/app/services/system_note_service.rb +++ b/app/services/system_note_service.rb @@ -22,8 +22,7 @@ module SystemNoteService commits_text = "#{total_count} commit".pluralize(total_count) body = "added #{commits_text}\n\n" - body << existing_commit_summary(noteable, existing_commits, oldrev) - body << new_commit_summary(new_commits).join("\n") + body << commits_list(noteable, new_commits, existing_commits, oldrev) body << "\n\n[Compare with previous version](#{diff_comparison_url(noteable, project, oldrev)})" create_note(NoteSummary.new(noteable, project, author, body, action: 'commit', commit_count: total_count)) @@ -481,7 +480,7 @@ module SystemNoteService # Returns an Array of Strings def new_commit_summary(new_commits) new_commits.collect do |commit| - "* #{commit.short_id} - #{escape_html(commit.title)}" + content_tag('li', "#{commit.short_id} - #{commit.title}") end end @@ -604,6 +603,16 @@ module SystemNoteService "#{cross_reference_note_prefix}#{gfm_reference}" end + # Builds a list of existing and new commits according to existing_commits and + # new_commits methods. + # Returns a String wrapped in `ul` and `li` tags. + def commits_list(noteable, new_commits, existing_commits, oldrev) + existing_commit_summary = existing_commit_summary(noteable, existing_commits, oldrev) + new_commit_summary = new_commit_summary(new_commits).join + + content_tag('ul', "#{existing_commit_summary}#{new_commit_summary}".html_safe) + end + # Build a single line summarizing existing commits being added in a merge # request # @@ -640,11 +649,8 @@ module SystemNoteService branch = noteable.target_branch branch = "#{noteable.target_project_namespace}:#{branch}" if noteable.for_fork? - "* #{commit_ids} - #{commits_text} from branch `#{branch}`\n" - end - - def escape_html(text) - Rack::Utils.escape_html(text) + branch_name = content_tag('code', branch) + content_tag('li', "#{commit_ids} - #{commits_text} from branch #{branch_name}".html_safe) end def url_helpers @@ -661,4 +667,8 @@ module SystemNoteService start_sha: oldrev ) end + + def content_tag(*args) + ActionController::Base.helpers.content_tag(*args) + end end diff --git a/changelogs/unreleased/25327-coverage-badge-rounding.yml b/changelogs/unreleased/25327-coverage-badge-rounding.yml new file mode 100644 index 00000000000..ea985689484 --- /dev/null +++ b/changelogs/unreleased/25327-coverage-badge-rounding.yml @@ -0,0 +1,5 @@ +--- +title: Show coverage to two decimal points in coverage badge +merge_request: 10083 +author: Jeff Stubler +type: changed diff --git a/changelogs/unreleased/39985-enable-prometheus-metrics-for-deployed-ingresses.yml b/changelogs/unreleased/39985-enable-prometheus-metrics-for-deployed-ingresses.yml new file mode 100644 index 00000000000..5c45d0db602 --- /dev/null +++ b/changelogs/unreleased/39985-enable-prometheus-metrics-for-deployed-ingresses.yml @@ -0,0 +1,5 @@ +--- +title: Enable Prometheus metrics for deployed Ingresses +merge_request: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/16866 +author: joshlambert +type: changed diff --git a/changelogs/unreleased/42270-fix-namespace-remove-exports-for-hashed-storage.yml b/changelogs/unreleased/42270-fix-namespace-remove-exports-for-hashed-storage.yml new file mode 100644 index 00000000000..d7a8b6e6f81 --- /dev/null +++ b/changelogs/unreleased/42270-fix-namespace-remove-exports-for-hashed-storage.yml @@ -0,0 +1,6 @@ +--- +title: Fix export removal for hashed-storage projects within a renamed or deleted + namespace +merge_request: 16658 +author: +type: fixed diff --git a/changelogs/unreleased/issue-42689-new-file-template.yml b/changelogs/unreleased/issue-42689-new-file-template.yml new file mode 100644 index 00000000000..d6b77b87605 --- /dev/null +++ b/changelogs/unreleased/issue-42689-new-file-template.yml @@ -0,0 +1,5 @@ +--- +title: Trigger change event on filename input when file template is applied +merge_request: 16911 +author: Sebastian Klingler +type: fixed diff --git a/changelogs/unreleased/osw-markdown-bypass-for-commit-messages.yml b/changelogs/unreleased/osw-markdown-bypass-for-commit-messages.yml new file mode 100644 index 00000000000..b2c1cd9710a --- /dev/null +++ b/changelogs/unreleased/osw-markdown-bypass-for-commit-messages.yml @@ -0,0 +1,5 @@ +--- +title: Bypass commits title markdown on notes +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/query-counts.yml b/changelogs/unreleased/query-counts.yml new file mode 100644 index 00000000000..e01ff8a4ad8 --- /dev/null +++ b/changelogs/unreleased/query-counts.yml @@ -0,0 +1,5 @@ +--- +title: Track and act upon the number of executed queries +merge_request: +author: +type: added diff --git a/config/initializers/query_limiting.rb b/config/initializers/query_limiting.rb new file mode 100644 index 00000000000..66864d1898e --- /dev/null +++ b/config/initializers/query_limiting.rb @@ -0,0 +1,9 @@ +if Gitlab::QueryLimiting.enable? + require_dependency 'gitlab/query_limiting/active_support_subscriber' + require_dependency 'gitlab/query_limiting/transaction' + require_dependency 'gitlab/query_limiting/middleware' + + Gitlab::Application.configure do |config| + config.middleware.use(Gitlab::QueryLimiting::Middleware) + end +end diff --git a/db/post_migrate/20171207150300_remove_project_labels_group_id_copy.rb b/db/post_migrate/20171207150300_remove_project_labels_group_id_copy.rb new file mode 100644 index 00000000000..2f339172eeb --- /dev/null +++ b/db/post_migrate/20171207150300_remove_project_labels_group_id_copy.rb @@ -0,0 +1,21 @@ +# Copy of 20180202111106 - this one should run before 20171207150343 to fix issues related to +# the removal of groups with labels. + +class RemoveProjectLabelsGroupIdCopy < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + # rubocop:disable Migration/UpdateColumnInBatches + update_column_in_batches(:labels, :group_id, nil) do |table, query| + query.where(table[:type].eq('ProjectLabel').and(table[:group_id].not_eq(nil))) + end + # rubocop:enable Migration/UpdateColumnInBatches + end + + def down + end +end diff --git a/doc/administration/environment_variables.md b/doc/administration/environment_variables.md index 9bcd13a52f7..e6c8f59549f 100644 --- a/doc/administration/environment_variables.md +++ b/doc/administration/environment_variables.md @@ -13,7 +13,7 @@ override certain values. Variable | Type | Description -------- | ---- | ----------- -`GITLAB_CDN_HOST` | string | Sets the hostname for a CDN to serve static assets (e.g. `mycdnsubdomain.fictional-cdn.com`) +`GITLAB_CDN_HOST` | string | Sets the base URL for a CDN to serve static assets (e.g. `//mycdnsubdomain.fictional-cdn.com`) `GITLAB_ROOT_PASSWORD` | string | Sets the password for the `root` user on installation `GITLAB_HOST` | string | The full URL of the GitLab server (including `http://` or `https://`) `RAILS_ENV` | string | The Rails environment; can be one of `production`, `development`, `staging` or `test` diff --git a/doc/development/README.md b/doc/development/README.md index 12cca9f84b7..45e9565f9a7 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -75,6 +75,7 @@ comments: false - [Ordering table columns](ordering_table_columns.md) - [Verifying database capabilities](verifying_database_capabilities.md) - [Database Debugging and Troubleshooting](database_debugging.md) +- [Query Count Limits](query_count_limits.md) ## Testing guides diff --git a/doc/development/query_count_limits.md b/doc/development/query_count_limits.md new file mode 100644 index 00000000000..ebb6e0c2dac --- /dev/null +++ b/doc/development/query_count_limits.md @@ -0,0 +1,65 @@ +# Query Count Limits + +Each controller or API endpoint is allowed to execute up to 100 SQL queries. In +a production environment we'll only log an error in case this threshold is +exceeded, but in a test environment we'll raise an error instead. + +## Solving Failing Tests + +When a test fails because it executes more than 100 SQL queries there are two +solutions to this problem: + +1. Reduce the number of SQL queries that are executed. +2. Whitelist the controller or API endpoint. + +You should only resort to whitelisting when an existing controller or endpoint +is to blame as in this case reducing the number of SQL queries can take a lot of +effort. Newly added controllers and endpoints are not allowed to execute more +than 100 SQL queries and no exceptions will be made for this rule. _If_ a large +number of SQL queries is necessary to perform certain work it's best to have +this work performed by Sidekiq instead of doing this directly in a web request. + +## Whitelisting + +In the event that you _have_ to whitelist a controller you'll first need to +create an issue. This issue should (preferably in the title) mention the +controller or endpoint and include the appropriate labels (`database`, +`performance`, and at least a team specific label such as `Discussion`). + +Once the issue has been created you can whitelist the code in question. For +Rails controllers it's best to create a `before_action` hook that runs as early +as possible. The called method in turn should call +`Gitlab::QueryLimiting.whitelist('issue URL here')`. For example: + +```ruby +class MyController < ApplicationController + before_action :whitelist_query_limiting, only: [:show] + + def index + # ... + end + + def show + # ... + end + + def whitelist_query_limiting + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/...') + end +end +``` + +By using a `before_action` you don't have to modify the controller method in +question, reducing the likelihood of merge conflicts. + +For Grape API endpoints there unfortunately is not a reliable way of running a +hook before a specific endpoint. This means that you have to add the whitelist +call directly into the endpoint like so: + +```ruby +get '/projects/:id/foo' do + Gitlab::QueryLimiting.whitelist('...') + + # ... +end +``` diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index 72def9d1d1d..8c639bd5343 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -46,7 +46,7 @@ namespace that started the import process. The importer will also import branches on forks of projects related to open pull requests. These branches will be imported with a naming scheme similar to -GH-SHA-Username/Pull-Request-number/fork-name/branch. This may lead to a discrepency +GH-SHA-Username/Pull-Request-number/fork-name/branch. This may lead to a discrepancy in branches compared to the GitHub Repository. For a more technical description and an overview of the architecture you can diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 9aeebc34525..c2113551207 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -42,7 +42,7 @@ module API include Gitlab::Auth::UserAuthFinders def find_current_user! - user = find_user_from_access_token || find_user_from_warden + user = find_user_from_sources return unless user forbidden!('User is blocked') unless Gitlab::UserAccess.new(user).allowed? && user.can?(:access_api) @@ -50,6 +50,10 @@ module API user end + def find_user_from_sources + find_user_from_access_token || find_user_from_warden + end + private # An array of scopes that were registered (using `allow_access_with_scope`) diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 0791a110c39..1794207e29b 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -29,6 +29,8 @@ module API use :pagination end get ':id/repository/branches' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42329') + repository = user_project.repository branches = ::Kaminari.paginate_array(repository.branches.sort_by(&:name)) merged_branch_names = repository.merged_branch_names(branches.map(&:name)) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index c99fe3ab5b3..b6c278c89d0 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -161,6 +161,8 @@ module API use :issue_params end post ':id/issues' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42320') + authorize! :create_issue, user_project # Setting created_at time only allowed for admins and project owners @@ -201,6 +203,8 @@ module API :labels, :created_at, :due_date, :confidential, :state_event end put ':id/issues/:issue_iid' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42322') + issue = user_project.issues.find_by!(iid: params.delete(:issue_iid)) authorize! :update_issue, issue @@ -234,6 +238,8 @@ module API requires :to_project_id, type: Integer, desc: 'The ID of the new project' end post ':id/issues/:issue_iid/move' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42323') + issue = user_project.issues.find_by(iid: params[:issue_iid]) not_found!('Issue') unless issue diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 420aaf1c964..719afa09295 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -152,6 +152,8 @@ module API use :optional_params end post ":id/merge_requests" do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42316') + authorize! :create_merge_request, user_project mr_params = declared_params(include_missing: false) @@ -256,6 +258,8 @@ module API at_least_one_of(*at_least_one_of_ce) end put ':id/merge_requests/:merge_request_iid' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42318') + merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request) mr_params = declared_params(include_missing: false) @@ -283,6 +287,8 @@ module API optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' end put ':id/merge_requests/:merge_request_iid/merge' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42317') + merge_request = find_project_merge_request(params[:merge_request_iid]) merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds]) diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 675c963bae2..d2b8b832e4e 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -42,6 +42,8 @@ module API requires :ref, type: String, desc: 'Reference' end post ':id/pipeline' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42124') + authorize! :create_pipeline, user_project new_pipeline = Ci::CreatePipelineService.new(user_project, diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 8b5e4f8edcc..5b481121a10 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -210,6 +210,8 @@ module API optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into' end post ':id/fork' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42284') + fork_params = declared_params(include_missing: false) namespace_id = fork_params[:namespace] diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index dd6801664b1..b3709455bc3 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -15,6 +15,8 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42283') + # validate variables params[:variables] = params[:variables].to_h unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } diff --git a/lib/api/users.rb b/lib/api/users.rb index e5de31ad51b..c7c2aa280d5 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -383,6 +383,8 @@ module API optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions" end delete ":id" do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42279') + authenticated_as_admin! user = User.find_by(id: params[:id]) diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb index b201bf77667..25176c5b38e 100644 --- a/lib/api/v3/branches.rb +++ b/lib/api/v3/branches.rb @@ -14,6 +14,8 @@ module API success ::API::Entities::Branch end get ":id/repository/branches" do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42276') + repository = user_project.repository branches = repository.branches.sort_by(&:name) merged_branch_names = repository.merged_branch_names(branches.map(&:name)) diff --git a/lib/api/v3/issues.rb b/lib/api/v3/issues.rb index cb371fdbab8..b59947d81d9 100644 --- a/lib/api/v3/issues.rb +++ b/lib/api/v3/issues.rb @@ -134,6 +134,8 @@ module API use :issue_params end post ':id/issues' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42131') + # Setting created_at time only allowed for admins and project owners unless current_user.admin? || user_project.owner == current_user params.delete(:created_at) @@ -169,6 +171,8 @@ module API :labels, :created_at, :due_date, :confidential, :state_event end put ':id/issues/:issue_id' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42132') + issue = user_project.issues.find(params.delete(:issue_id)) authorize! :update_issue, issue @@ -201,6 +205,8 @@ module API requires :to_project_id, type: Integer, desc: 'The ID of the new project' end post ':id/issues/:issue_id/move' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42133') + issue = user_project.issues.find_by(id: params[:issue_id]) not_found!('Issue') unless issue diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index 0a24fea52a3..ce216497996 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -91,6 +91,8 @@ module API use :optional_params end post ":id/merge_requests" do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42126') + authorize! :create_merge_request, user_project mr_params = declared_params(include_missing: false) @@ -167,6 +169,8 @@ module API :remove_source_branch end put path do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42127') + merge_request = find_merge_request_with_access(params.delete(:merge_request_id), :update_merge_request) mr_params = declared_params(include_missing: false) diff --git a/lib/api/v3/pipelines.rb b/lib/api/v3/pipelines.rb index c48cbd2b765..6d31c12f572 100644 --- a/lib/api/v3/pipelines.rb +++ b/lib/api/v3/pipelines.rb @@ -19,6 +19,8 @@ module API desc: 'Either running, branches, or tags' end get ':id/pipelines' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42123') + authorize! :read_pipeline, user_project pipelines = PipelinesFinder.new(user_project, scope: params[:scope]).execute diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index 534911fde5c..34f07dfb486 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -16,6 +16,8 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/builds", requirements: { ref: /.+/ } do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42121') + # validate variables params[:variables] = params[:variables].to_h unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 9a0482306b7..778d78185ff 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -23,7 +23,7 @@ module Gitlab @coverage ||= raw_coverage return unless @coverage - @coverage.to_i + @coverage.to_f.round(2) end def metadata diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb index fcecb1d9665..afbf9dd17e3 100644 --- a/lib/gitlab/badge/coverage/template.rb +++ b/lib/gitlab/badge/coverage/template.rb @@ -25,7 +25,7 @@ module Gitlab end def value_text - @status ? "#{@status}%" : 'unknown' + @status ? ("%.2f%%" % @status) : 'unknown' end def key_width @@ -33,7 +33,7 @@ module Gitlab end def value_width - @status ? 36 : 58 + @status ? 54 : 58 end def value_color diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 3487e099381..ae7e88f0503 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: no RPC's here. - module Gitlab module Git class Branch < Ref diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index f28624ff37a..ab1362a3bb0 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1222,33 +1222,13 @@ module Gitlab end def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) - squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) - env = git_env_for_user(user).merge( - 'GIT_AUTHOR_NAME' => author.name, - 'GIT_AUTHOR_EMAIL' => author.email - ) - diff_range = "#{start_sha}...#{end_sha}" - diff_files = run_git!( - %W(diff --name-only --diff-filter=a --binary #{diff_range}) - ).chomp - - with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do - # Apply diff of the `diff_range` to the worktree - diff = run_git!(%W(diff --binary #{diff_range})) - run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| - stdin.write(diff) + gitaly_migrate(:squash) do |is_enabled| + if is_enabled + gitaly_operation_client.user_squash(user, squash_id, branch, + start_sha, end_sha, author, message) + else + git_squash(user, squash_id, branch, start_sha, end_sha, author, message) end - - # Commit the `diff_range` diff - run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) - - # Return the squash sha. May print a warning for ambiguous refs, but - # we can ignore that with `--quiet` and just take the SHA, if present. - # HEAD here always refers to the current HEAD commit, even if there is - # another ref called HEAD. - run_git!( - %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env - ).chomp end end @@ -1363,20 +1343,23 @@ module Gitlab raise CommandError.new(e) end - def refs_contains_sha(ref_type, sha) - args = %W(#{ref_type} --contains #{sha}) - names = run_git(args).first - - if names.respond_to?(:split) - names = names.split("\n").map(&:strip) - - names.each do |name| - name.slice! '* ' + def branch_names_contains_sha(sha) + gitaly_migrate(:branch_names_contains_sha) do |is_enabled| + if is_enabled + gitaly_ref_client.branch_names_contains_sha(sha) + else + refs_contains_sha(:branch, sha) end + end + end - names - else - [] + def tag_names_contains_sha(sha) + gitaly_migrate(:tag_names_contains_sha) do |is_enabled| + if is_enabled + gitaly_ref_client.tag_names_contains_sha(sha) + else + refs_contains_sha(:tag, sha) + end end end @@ -1454,6 +1437,21 @@ module Gitlab end end + def refs_contains_sha(ref_type, sha) + args = %W(#{ref_type} --contains #{sha}) + names = run_git(args).first + + return [] unless names.respond_to?(:split) + + names = names.split("\n").map(&:strip) + + names.each do |name| + name.slice! '* ' + end + + names + end + def rugged_write_config(full_path:) rugged.config['gitlab.fullpath'] = full_path end @@ -2164,6 +2162,37 @@ module Gitlab end end + def git_squash(user, squash_id, branch, start_sha, end_sha, author, message) + squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) + env = git_env_for_user(user).merge( + 'GIT_AUTHOR_NAME' => author.name, + 'GIT_AUTHOR_EMAIL' => author.email + ) + diff_range = "#{start_sha}...#{end_sha}" + diff_files = run_git!( + %W(diff --name-only --diff-filter=a --binary #{diff_range}) + ).chomp + + with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do + # Apply diff of the `diff_range` to the worktree + diff = run_git!(%W(diff --binary #{diff_range})) + run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| + stdin.write(diff) + end + + # Commit the `diff_range` diff + run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) + + # Return the squash sha. May print a warning for ambiguous refs, but + # we can ignore that with `--quiet` and just take the SHA, if present. + # HEAD here always refers to the current HEAD commit, even if there is + # another ref called HEAD. + run_git!( + %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env + ).chomp + end + end + def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index bc4e160dce9..8a8f7b051ed 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: no RPC's here. -# module Gitlab module Git class Tag < Ref diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index cd2734b5a07..831cfd1e014 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -183,6 +183,32 @@ module Gitlab end end + def user_squash(user, squash_id, branch, start_sha, end_sha, author, message) + request = Gitaly::UserSquashRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + squash_id: squash_id.to_s, + branch: encode_binary(branch), + start_sha: start_sha, + end_sha: end_sha, + author: Gitlab::Git::User.from_gitlab(author).to_gitaly, + commit_message: encode_binary(message) + ) + + response = GitalyClient.call( + @repository.storage, + :operation_service, + :user_squash, + request + ) + + if response.git_error.presence + raise Gitlab::Git::Repository::GitError, response.git_error + end + + response.squash_sha + end + def user_commit_files( user, branch_name, commit_message, actions, author_email, author_name, start_branch_name, start_repository) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 8b9a224b700..ba6b577fd17 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -145,6 +145,32 @@ module Gitlab raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? end + # Limit: 0 implies no limit, thus all tag names will be returned + def tag_names_contains_sha(sha, limit: 0) + request = Gitaly::ListTagNamesContainingCommitRequest.new( + repository: @gitaly_repo, + commit_id: sha, + limit: limit + ) + + stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request) + + consume_ref_contains_sha_response(stream, :tag_names) + end + + # Limit: 0 implies no limit, thus all tag names will be returned + def branch_names_contains_sha(sha, limit: 0) + request = Gitaly::ListBranchNamesContainingCommitRequest.new( + repository: @gitaly_repo, + commit_id: sha, + limit: limit + ) + + stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request) + + consume_ref_contains_sha_response(stream, :branch_names) + end + private def consume_refs_response(response) @@ -215,6 +241,13 @@ module Gitlab Gitlab::Git::Commit.decorate(@repository, hash) end + def consume_ref_contains_sha_response(stream, collection_name) + stream.each_with_object([]) do |response, array| + encoded_names = response.send(collection_name).map { |b| Gitlab::Git.ref_name(b) } # rubocop:disable GitlabSecurity/PublicSend + array.concat(encoded_names) + end + end + def invalid_ref!(message) raise Gitlab::Git::Repository::InvalidRef.new(message) end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index a3216759cae..ca5e06009fa 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -64,7 +64,7 @@ module Gitlab { name: 'configuration-volume', configMap: { - name: 'values-content-configuration', + name: "values-content-configuration-#{command.name}", items: [{ key: 'values', path: 'values.yaml' }] } } @@ -81,7 +81,11 @@ module Gitlab def create_config_map resource = ::Kubeclient::Resource.new - resource.metadata = { name: 'values-content-configuration', namespace: namespace_name, labels: { name: 'values-content-configuration' } } + resource.metadata = { + name: "values-content-configuration-#{command.name}", + namespace: namespace_name, + labels: { name: "values-content-configuration-#{command.name}" } + } resource.data = { values: File.read(command.chart_values_file) } kubeclient.create_config_map(resource) end diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb new file mode 100644 index 00000000000..f64f1757144 --- /dev/null +++ b/lib/gitlab/query_limiting.rb @@ -0,0 +1,36 @@ +module Gitlab + module QueryLimiting + # Returns true if we should enable tracking of query counts. + # + # This is only enabled in production/staging if we're running on GitLab.com. + # This ensures we don't produce any errors that users can't do anything + # about themselves. + def self.enable? + Gitlab.com? || Rails.env.development? || Rails.env.test? + end + + # Allows the current request to execute any number of SQL queries. + # + # This method should _only_ be used when there's a corresponding issue to + # reduce the number of queries. + # + # The issue URL is only meant to push developers into creating an issue + # instead of blindly whitelisting offending blocks of code. + def self.whitelist(issue_url) + return unless enable_whitelist? + + unless issue_url.start_with?('https://') + raise( + ArgumentError, + 'You must provide a valid issue URL in order to whitelist a block of code' + ) + end + + Transaction&.current&.whitelisted = true + end + + def self.enable_whitelist? + Rails.env.development? || Rails.env.test? + end + end +end diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb new file mode 100644 index 00000000000..66049c94ec6 --- /dev/null +++ b/lib/gitlab/query_limiting/active_support_subscriber.rb @@ -0,0 +1,11 @@ +module Gitlab + module QueryLimiting + class ActiveSupportSubscriber < ActiveSupport::Subscriber + attach_to :active_record + + def sql(*) + Transaction.current&.increment + end + end + end +end diff --git a/lib/gitlab/query_limiting/middleware.rb b/lib/gitlab/query_limiting/middleware.rb new file mode 100644 index 00000000000..949ae79a047 --- /dev/null +++ b/lib/gitlab/query_limiting/middleware.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module QueryLimiting + # Middleware for reporting (or raising) when a request performs more than a + # certain amount of database queries. + class Middleware + CONTROLLER_KEY = 'action_controller.instance'.freeze + ENDPOINT_KEY = 'api.endpoint'.freeze + + def initialize(app) + @app = app + end + + def call(env) + transaction, retval = Transaction.run do + @app.call(env) + end + + transaction.action = action_name(env) + transaction.act_upon_results + + retval + end + + def action_name(env) + if env[CONTROLLER_KEY] + action_for_rails(env) + elsif env[ENDPOINT_KEY] + action_for_grape(env) + end + end + + private + + def action_for_rails(env) + controller = env[CONTROLLER_KEY] + action = "#{controller.class.name}##{controller.action_name}" + + if controller.content_type == 'text/html' + action + else + "#{action} (#{controller.content_type})" + end + end + + def action_for_grape(env) + endpoint = env[ENDPOINT_KEY] + route = endpoint.route rescue nil + + "#{route.request_method} #{route.path}" if route + end + end + end +end diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb new file mode 100644 index 00000000000..3cbafadb0d0 --- /dev/null +++ b/lib/gitlab/query_limiting/transaction.rb @@ -0,0 +1,83 @@ +module Gitlab + module QueryLimiting + class Transaction + THREAD_KEY = :__gitlab_query_counts_transaction + + attr_accessor :count, :whitelisted + + # The name of the action (e.g. `UsersController#show`) that is being + # executed. + attr_accessor :action + + # The maximum number of SQL queries that can be executed in a request. For + # the sake of keeping things simple we hardcode this value here, it's not + # supposed to be changed very often anyway. + THRESHOLD = 100 + + # Error that is raised whenever exceeding the maximum number of queries. + ThresholdExceededError = Class.new(StandardError) + + def self.current + Thread.current[THREAD_KEY] + end + + # Starts a new transaction and returns it and the blocks' return value. + # + # Example: + # + # transaction, retval = Transaction.run do + # 10 + # end + # + # retval # => 10 + def self.run + transaction = new + Thread.current[THREAD_KEY] = transaction + + [transaction, yield] + ensure + Thread.current[THREAD_KEY] = nil + end + + def initialize + @action = nil + @count = 0 + @whitelisted = false + end + + # Sends a notification based on the number of executed SQL queries. + def act_upon_results + return unless threshold_exceeded? + + error = ThresholdExceededError.new(error_message) + + if raise_error? + raise(error) + else + # Raven automatically logs to the Rails log if disabled, thus we don't + # need to manually log anything in case Sentry support is not enabled. + Raven.capture_exception(error) + end + end + + def increment + @count += 1 unless whitelisted + end + + def raise_error? + Rails.env.test? + end + + def threshold_exceeded? + count > THRESHOLD + end + + def error_message + header = 'Too many SQL queries were executed' + header += " in #{action}" if action + + "#{header}: a maximum of #{THRESHOLD} is allowed but #{count} SQL queries were executed" + end + end + end +end diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake index b1e012e70c5..4b4881cecb8 100644 --- a/lib/tasks/flay.rake +++ b/lib/tasks/flay.rake @@ -2,7 +2,7 @@ desc 'Code duplication analyze via flay' task :flay do output = `bundle exec flay --mass 35 app/ lib/gitlab/ 2> #{File::NULL}` - if output.include? "Similar code found" + if output.include?("Similar code found") || output.include?("IDENTICAL code found") puts output exit 1 end diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb index 16d328a5bc2..20976977f21 100644 --- a/spec/factories/projects.rb +++ b/spec/factories/projects.rb @@ -93,6 +93,12 @@ FactoryBot.define do avatar { fixture_file_upload('spec/fixtures/dk.png') } end + trait :with_export do + after(:create) do |project, evaluator| + ProjectExportWorker.new.perform(project.creator.id, project.id) + end + end + trait :broken_storage do after(:create) do |project| project.update_column(:repository_storage, 'broken') diff --git a/spec/features/projects/badges/coverage_spec.rb b/spec/features/projects/badges/coverage_spec.rb index 821ce88a402..f51001edcd7 100644 --- a/spec/features/projects/badges/coverage_spec.rb +++ b/spec/features/projects/badges/coverage_spec.rb @@ -18,7 +18,7 @@ feature 'test coverage badge' do show_test_coverage_badge - expect_coverage_badge('95%') + expect_coverage_badge('95.00%') end scenario 'user requests coverage badge for specific job' do @@ -30,7 +30,7 @@ feature 'test coverage badge' do show_test_coverage_badge(job: 'coverage') - expect_coverage_badge('85%') + expect_coverage_badge('85.00%') end scenario 'user requests coverage badge for pipeline without coverage' do diff --git a/spec/features/projects/import_export/namespace_export_file_spec.rb b/spec/features/projects/import_export/namespace_export_file_spec.rb index e76bc6f1220..c1fccf4a40b 100644 --- a/spec/features/projects/import_export/namespace_export_file_spec.rb +++ b/spec/features/projects/import_export/namespace_export_file_spec.rb @@ -1,44 +1,37 @@ require 'spec_helper' feature 'Import/Export - Namespace export file cleanup', :js do - let(:export_path) { "#{Dir.tmpdir}/import_file_spec" } - let(:config_hash) { YAML.load_file(Gitlab::ImportExport.config_file).deep_stringify_keys } + let(:export_path) { Dir.mktmpdir('namespace_export_file_spec') } - let(:project) { create(:project) } - - background do - allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) + before do + allow(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) end after do FileUtils.rm_rf(export_path, secure: true) end - context 'admin user' do + shared_examples_for 'handling project exports on namespace change' do + let!(:old_export_path) { project.export_path } + before do sign_in(create(:admin)) + + setup_export_project end context 'moving the namespace' do - scenario 'removes the export file' do - setup_export_project - - old_export_path = project.export_path.dup - + it 'removes the export file' do expect(File).to exist(old_export_path) - project.namespace.update(path: 'new_path') + project.namespace.update!(path: build(:namespace).path) expect(File).not_to exist(old_export_path) end end context 'deleting the namespace' do - scenario 'removes the export file' do - setup_export_project - - old_export_path = project.export_path.dup - + it 'removes the export file' do expect(File).to exist(old_export_path) project.namespace.destroy @@ -46,17 +39,29 @@ feature 'Import/Export - Namespace export file cleanup', :js do expect(File).not_to exist(old_export_path) end end + end - def setup_export_project - visit edit_project_path(project) + describe 'legacy storage' do + let(:project) { create(:project) } - expect(page).to have_content('Export project') + it_behaves_like 'handling project exports on namespace change' + end + + describe 'hashed storage' do + let(:project) { create(:project, :hashed) } - find(:link, 'Export project').send_keys(:return) + it_behaves_like 'handling project exports on namespace change' + end - visit edit_project_path(project) + def setup_export_project + visit edit_project_path(project) - expect(page).to have_content('Download export') - end + expect(page).to have_content('Export project') + + find(:link, 'Export project').send_keys(:return) + + visit edit_project_path(project) + + expect(page).to have_content('Download export') end end diff --git a/spec/lib/gitlab/badge/coverage/template_spec.rb b/spec/lib/gitlab/badge/coverage/template_spec.rb index 383bae6e087..d9c21a22590 100644 --- a/spec/lib/gitlab/badge/coverage/template_spec.rb +++ b/spec/lib/gitlab/badge/coverage/template_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Gitlab::Badge::Coverage::Template do - let(:badge) { double(entity: 'coverage', status: 90) } + let(:badge) { double(entity: 'coverage', status: 90.00) } let(:template) { described_class.new(badge) } describe '#key_text' do @@ -13,7 +13,17 @@ describe Gitlab::Badge::Coverage::Template do describe '#value_text' do context 'when coverage is known' do it 'returns coverage percentage' do - expect(template.value_text).to eq '90%' + expect(template.value_text).to eq '90.00%' + end + end + + context 'when coverage is known to many digits' do + before do + allow(badge).to receive(:status).and_return(92.349) + end + + it 'returns rounded coverage percentage' do + expect(template.value_text).to eq '92.35%' end end @@ -37,7 +47,7 @@ describe Gitlab::Badge::Coverage::Template do describe '#value_width' do context 'when coverage is known' do it 'is narrower when coverage is known' do - expect(template.value_width).to eq 36 + expect(template.value_width).to eq 54 end end @@ -113,7 +123,7 @@ describe Gitlab::Badge::Coverage::Template do describe '#width' do context 'when coverage is known' do it 'returns the key width plus value width' do - expect(template.width).to eq 98 + expect(template.width).to eq 116 end end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 96a442f782f..ec1c7a96f92 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -1162,14 +1162,27 @@ describe Gitlab::Git::Repository, seed_helper: true do context 'when Gitaly find_branch feature is disabled', :skip_gitaly_mock do it_behaves_like 'finding a branch' - it 'should reload Rugged::Repository and return master' do - expect(Rugged::Repository).to receive(:new).twice.and_call_original + context 'force_reload is true' do + it 'should reload Rugged::Repository' do + expect(Rugged::Repository).to receive(:new).twice.and_call_original - repository.find_branch('master') - branch = repository.find_branch('master', force_reload: true) + repository.find_branch('master') + branch = repository.find_branch('master', force_reload: true) - expect(branch).to be_a_kind_of(Gitlab::Git::Branch) - expect(branch.name).to eq('master') + expect(branch).to be_a_kind_of(Gitlab::Git::Branch) + expect(branch.name).to eq('master') + end + end + + context 'force_reload is false' do + it 'should not reload Rugged::Repository' do + expect(Rugged::Repository).to receive(:new).once.and_call_original + + branch = repository.find_branch('master', force_reload: false) + + expect(branch).to be_a_kind_of(Gitlab::Git::Branch) + expect(branch.name).to eq('master') + end end end end @@ -2183,7 +2196,7 @@ describe Gitlab::Git::Repository, seed_helper: true do repository.squash(user, squash_id, opts) end - context 'sparse checkout' do + context 'sparse checkout', :skip_gitaly_mock do let(:expected_files) { %w(files files/js files/js/application.js) } before do diff --git a/spec/lib/gitlab/gitaly_client/operation_service_spec.rb b/spec/lib/gitlab/gitaly_client/operation_service_spec.rb index d9ec28ab02e..9fbdd73ee0e 100644 --- a/spec/lib/gitlab/gitaly_client/operation_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/operation_service_spec.rb @@ -123,4 +123,53 @@ describe Gitlab::GitalyClient::OperationService do expect(subject.branch_created).to be(false) end end + + describe '#user_squash' do + let(:branch_name) { 'my-branch' } + let(:squash_id) { '1' } + let(:start_sha) { 'b83d6e391c22777fca1ed3012fce84f633d7fed0' } + let(:end_sha) { '54cec5282aa9f21856362fe321c800c236a61615' } + let(:commit_message) { 'Squash message' } + let(:request) do + Gitaly::UserSquashRequest.new( + repository: repository.gitaly_repository, + user: gitaly_user, + squash_id: squash_id.to_s, + branch: branch_name, + start_sha: start_sha, + end_sha: end_sha, + author: gitaly_user, + commit_message: commit_message + ) + end + let(:squash_sha) { 'f00' } + let(:response) { Gitaly::UserSquashResponse.new(squash_sha: squash_sha) } + + subject do + client.user_squash(user, squash_id, branch_name, start_sha, end_sha, user, commit_message) + end + + it 'sends a user_squash message and returns the squash sha' do + expect_any_instance_of(Gitaly::OperationService::Stub) + .to receive(:user_squash).with(request, kind_of(Hash)) + .and_return(response) + + expect(subject).to eq(squash_sha) + end + + context "when git_error is present" do + let(:response) do + Gitaly::UserSquashResponse.new(git_error: "something failed") + end + + it "throws a PreReceive exception" do + expect_any_instance_of(Gitaly::OperationService::Stub) + .to receive(:user_squash).with(request, kind_of(Hash)) + .and_return(response) + + expect { subject }.to raise_error( + Gitlab::Git::Repository::GitError, "something failed") + end + end + end end diff --git a/spec/lib/gitlab/kubernetes/helm/pod_spec.rb b/spec/lib/gitlab/kubernetes/helm/pod_spec.rb index 0b8e97b8948..ebb6033f71e 100644 --- a/spec/lib/gitlab/kubernetes/helm/pod_spec.rb +++ b/spec/lib/gitlab/kubernetes/helm/pod_spec.rb @@ -63,14 +63,14 @@ describe Gitlab::Kubernetes::Helm::Pod do it 'should mount configMap specification in the volume' do spec = subject.generate.spec - expect(spec.volumes.first.configMap['name']).to eq('values-content-configuration') + expect(spec.volumes.first.configMap['name']).to eq("values-content-configuration-#{app.name}") expect(spec.volumes.first.configMap['items'].first['key']).to eq('values') expect(spec.volumes.first.configMap['items'].first['path']).to eq('values.yaml') end end context 'without a configuration file' do - let(:app) { create(:clusters_applications_ingress, cluster: cluster) } + let(:app) { create(:clusters_applications_helm, cluster: cluster) } it_behaves_like 'helm pod' diff --git a/spec/lib/gitlab/query_limiting/active_support_subscriber_spec.rb b/spec/lib/gitlab/query_limiting/active_support_subscriber_spec.rb new file mode 100644 index 00000000000..b49bc5c328c --- /dev/null +++ b/spec/lib/gitlab/query_limiting/active_support_subscriber_spec.rb @@ -0,0 +1,19 @@ +require 'spec_helper' + +describe Gitlab::QueryLimiting::ActiveSupportSubscriber do + describe '#sql' do + it 'increments the number of executed SQL queries' do + transaction = double(:transaction) + + allow(Gitlab::QueryLimiting::Transaction) + .to receive(:current) + .and_return(transaction) + + expect(transaction) + .to receive(:increment) + .at_least(:once) + + User.count + end + end +end diff --git a/spec/lib/gitlab/query_limiting/middleware_spec.rb b/spec/lib/gitlab/query_limiting/middleware_spec.rb new file mode 100644 index 00000000000..a04bcdecb4b --- /dev/null +++ b/spec/lib/gitlab/query_limiting/middleware_spec.rb @@ -0,0 +1,72 @@ +require 'spec_helper' + +describe Gitlab::QueryLimiting::Middleware do + describe '#call' do + it 'runs the application with query limiting in place' do + middleware = described_class.new(-> (env) { env }) + + expect_any_instance_of(Gitlab::QueryLimiting::Transaction) + .to receive(:act_upon_results) + + expect(middleware.call({ number: 10 })) + .to eq({ number: 10 }) + end + end + + describe '#action_name' do + let(:middleware) { described_class.new(-> (env) { env }) } + + context 'using a Rails request' do + it 'returns the name of the controller and action' do + env = { + described_class::CONTROLLER_KEY => double( + :controller, + action_name: 'show', + class: double(:class, name: 'UsersController'), + content_type: 'text/html' + ) + } + + expect(middleware.action_name(env)).to eq('UsersController#show') + end + + it 'includes the content type if this is not text/html' do + env = { + described_class::CONTROLLER_KEY => double( + :controller, + action_name: 'show', + class: double(:class, name: 'UsersController'), + content_type: 'application/json' + ) + } + + expect(middleware.action_name(env)) + .to eq('UsersController#show (application/json)') + end + end + + context 'using a Grape API request' do + it 'returns the name of the request method and endpoint path' do + env = { + described_class::ENDPOINT_KEY => double( + :endpoint, + route: double(:route, request_method: 'GET', path: '/foo') + ) + } + + expect(middleware.action_name(env)).to eq('GET /foo') + end + + it 'returns nil if the route can not be retrieved' do + endpoint = double(:endpoint) + env = { described_class::ENDPOINT_KEY => endpoint } + + allow(endpoint) + .to receive(:route) + .and_raise(RuntimeError) + + expect(middleware.action_name(env)).to be_nil + end + end + end +end diff --git a/spec/lib/gitlab/query_limiting/transaction_spec.rb b/spec/lib/gitlab/query_limiting/transaction_spec.rb new file mode 100644 index 00000000000..b4231fcd0fa --- /dev/null +++ b/spec/lib/gitlab/query_limiting/transaction_spec.rb @@ -0,0 +1,144 @@ +require 'spec_helper' + +describe Gitlab::QueryLimiting::Transaction do + after do + Thread.current[described_class::THREAD_KEY] = nil + end + + describe '.current' do + it 'returns nil when there is no transaction' do + expect(described_class.current).to be_nil + end + + it 'returns the transaction when present' do + Thread.current[described_class::THREAD_KEY] = described_class.new + + expect(described_class.current).to be_an_instance_of(described_class) + end + end + + describe '.run' do + it 'runs a transaction and returns it and its return value' do + trans, ret = described_class.run do + 10 + end + + expect(trans).to be_an_instance_of(described_class) + expect(ret).to eq(10) + end + + it 'removes the transaction from the current thread upon completion' do + described_class.run do + 10 + end + + expect(Thread.current[described_class::THREAD_KEY]).to be_nil + end + end + + describe '#act_upon_results' do + context 'when the query threshold is not exceeded' do + it 'does nothing' do + trans = described_class.new + + expect(trans).not_to receive(:raise) + + trans.act_upon_results + end + end + + context 'when the query threshold is exceeded' do + let(:transaction) do + trans = described_class.new + trans.count = described_class::THRESHOLD + 1 + + trans + end + + it 'raises an error when this is enabled' do + expect { transaction.act_upon_results } + .to raise_error(described_class::ThresholdExceededError) + end + + it 'reports the error in Sentry if raising an error is disabled' do + expect(transaction) + .to receive(:raise_error?) + .and_return(false) + + expect(Raven) + .to receive(:capture_exception) + .with(an_instance_of(described_class::ThresholdExceededError)) + + transaction.act_upon_results + end + end + end + + describe '#increment' do + it 'increments the number of executed queries' do + transaction = described_class.new + + expect(transaction.count).to be_zero + + transaction.increment + + expect(transaction.count).to eq(1) + end + end + + describe '#raise_error?' do + it 'returns true in a test environment' do + transaction = described_class.new + + expect(transaction.raise_error?).to eq(true) + end + + it 'returns false in a production environment' do + transaction = described_class.new + + expect(Rails.env) + .to receive(:test?) + .and_return(false) + + expect(transaction.raise_error?).to eq(false) + end + end + + describe '#threshold_exceeded?' do + it 'returns false when the threshold is not exceeded' do + transaction = described_class.new + + expect(transaction.threshold_exceeded?).to eq(false) + end + + it 'returns true when the threshold is exceeded' do + transaction = described_class.new + transaction.count = described_class::THRESHOLD + 1 + + expect(transaction.threshold_exceeded?).to eq(true) + end + end + + describe '#error_message' do + it 'returns the error message to display when the threshold is exceeded' do + transaction = described_class.new + transaction.count = max = described_class::THRESHOLD + + expect(transaction.error_message).to eq( + "Too many SQL queries were executed: a maximum of #{max} " \ + "is allowed but #{max} SQL queries were executed" + ) + end + + it 'includes the action name in the error message when present' do + transaction = described_class.new + transaction.count = max = described_class::THRESHOLD + transaction.action = 'UsersController#show' + + expect(transaction.error_message).to eq( + "Too many SQL queries were executed in UsersController#show: " \ + "a maximum of #{max} is allowed but #{max} SQL queries were executed" + ) + end + end +end diff --git a/spec/lib/gitlab/query_limiting_spec.rb b/spec/lib/gitlab/query_limiting_spec.rb new file mode 100644 index 00000000000..2eddab0b8c3 --- /dev/null +++ b/spec/lib/gitlab/query_limiting_spec.rb @@ -0,0 +1,65 @@ +require 'spec_helper' + +describe Gitlab::QueryLimiting do + describe '.enable?' do + it 'returns true in a test environment' do + expect(described_class.enable?).to eq(true) + end + + it 'returns true in a development environment' do + allow(Rails.env).to receive(:development?).and_return(true) + + expect(described_class.enable?).to eq(true) + end + + it 'returns true on GitLab.com' do + allow(Gitlab).to receive(:com?).and_return(true) + + expect(described_class.enable?).to eq(true) + end + + it 'returns true in a non GitLab.com' do + expect(Gitlab).to receive(:com?).and_return(false) + expect(Rails.env).to receive(:development?).and_return(false) + expect(Rails.env).to receive(:test?).and_return(false) + + expect(described_class.enable?).to eq(false) + end + end + + describe '.whitelist' do + it 'raises ArgumentError when an invalid issue URL is given' do + expect { described_class.whitelist('foo') } + .to raise_error(ArgumentError) + end + + context 'without a transaction' do + it 'does nothing' do + expect { described_class.whitelist('https://example.com') } + .not_to raise_error + end + end + + context 'with a transaction' do + let(:transaction) { Gitlab::QueryLimiting::Transaction.new } + + before do + allow(Gitlab::QueryLimiting::Transaction) + .to receive(:current) + .and_return(transaction) + end + + it 'does not increment the number of SQL queries executed in the block' do + before = transaction.count + + described_class.whitelist('https://example.com') + + 2.times do + User.count + end + + expect(transaction.count).to eq(before) + end + end + end +end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 6b7dbad128c..824cca66fb4 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -596,4 +596,24 @@ describe Namespace do end end end + + describe '#remove_exports' do + let(:legacy_project) { create(:project, :with_export, namespace: namespace) } + let(:hashed_project) { create(:project, :with_export, :hashed, namespace: namespace) } + let(:export_path) { Dir.mktmpdir('namespace_remove_exports_spec') } + let(:legacy_export) { legacy_project.export_project_path } + let(:hashed_export) { hashed_project.export_project_path } + + it 'removes exports for legacy and hashed projects' do + allow(Gitlab::ImportExport).to receive(:storage_path) { export_path } + + expect(File.exist?(legacy_export)).to be_truthy + expect(File.exist?(hashed_export)).to be_truthy + + namespace.remove_exports! + + expect(File.exist?(legacy_export)).to be_falsy + expect(File.exist?(hashed_export)).to be_falsy + end + end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index df4c796c61e..da940571bc1 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2503,6 +2503,37 @@ describe Project do end end + describe '#remove_exports' do + let(:project) { create(:project, :with_export) } + + it 'removes the exports directory for the project' do + expect(File.exist?(project.export_path)).to be_truthy + + allow(FileUtils).to receive(:rm_rf).and_call_original + expect(FileUtils).to receive(:rm_rf).with(project.export_path).and_call_original + project.remove_exports + + expect(File.exist?(project.export_path)).to be_falsy + end + + it 'is a no-op when there is no namespace' do + export_path = project.export_path + project.update_column(:namespace_id, nil) + + expect(FileUtils).not_to receive(:rm_rf).with(export_path) + + project.remove_exports + + expect(File.exist?(export_path)).to be_truthy + end + + it 'is run when the project is destroyed' do + expect(project).to receive(:remove_exports).and_call_original + + project.destroy + end + end + describe '#forks_count' do it 'returns the number of forks' do project = build(:project) diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index 1102b1c9006..02a5ee54262 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -36,26 +36,49 @@ describe Repository do end describe '#branch_names_contains' do - subject { repository.branch_names_contains(sample_commit.id) } + shared_examples '#branch_names_contains' do + set(:project) { create(:project, :repository) } + let(:repository) { project.repository } - it { is_expected.to include('master') } - it { is_expected.not_to include('feature') } - it { is_expected.not_to include('fix') } + subject { repository.branch_names_contains(sample_commit.id) } - describe 'when storage is broken', :broken_storage do - it 'should raise a storage error' do - expect_to_raise_storage_error do - broken_repository.branch_names_contains(sample_commit.id) + it { is_expected.to include('master') } + it { is_expected.not_to include('feature') } + it { is_expected.not_to include('fix') } + + describe 'when storage is broken', :broken_storage do + it 'should raise a storage error' do + expect_to_raise_storage_error do + broken_repository.branch_names_contains(sample_commit.id) + end end end end + + context 'when gitaly is enabled' do + it_behaves_like '#branch_names_contains' + end + + context 'when gitaly is disabled', :skip_gitaly_mock do + it_behaves_like '#branch_names_contains' + end end describe '#tag_names_contains' do - subject { repository.tag_names_contains(sample_commit.id) } + shared_examples '#tag_names_contains' do + subject { repository.tag_names_contains(sample_commit.id) } - it { is_expected.to include('v1.1.0') } - it { is_expected.not_to include('v1.0.0') } + it { is_expected.to include('v1.1.0') } + it { is_expected.not_to include('v1.0.0') } + end + + context 'when gitaly is enabled' do + it_behaves_like '#tag_names_contains' + end + + context 'when gitaly is enabled', :skip_gitaly_mock do + it_behaves_like '#tag_names_contains' + end end describe 'tags_sorted_by' do @@ -959,19 +982,19 @@ describe Repository do end describe '#find_branch' do - it 'loads a branch with a fresh repo' do - expect(Gitlab::Git::Repository).to receive(:new).twice.and_call_original + context 'fresh_repo is true' do + it 'delegates the call to raw_repository' do + expect(repository.raw_repository).to receive(:find_branch).with('master', true) - 2.times do - expect(repository.find_branch('feature')).not_to be_nil + repository.find_branch('master', fresh_repo: true) end end - it 'loads a branch with a cached repo' do - expect(Gitlab::Git::Repository).to receive(:new).once.and_call_original + context 'fresh_repo is false' do + it 'delegates the call to raw_repository' do + expect(repository.raw_repository).to receive(:find_branch).with('master', false) - 2.times do - expect(repository.find_branch('feature', fresh_repo: false)).not_to be_nil + repository.find_branch('master', fresh_repo: false) end end end diff --git a/spec/requests/api/groups_spec.rb b/spec/requests/api/groups_spec.rb index 3c0b4728dc2..bb0034e3237 100644 --- a/spec/requests/api/groups_spec.rb +++ b/spec/requests/api/groups_spec.rb @@ -30,6 +30,21 @@ describe API::Groups do expect(json_response) .to satisfy_one { |group| group['name'] == group1.name } end + + it 'avoids N+1 queries' do + # Establish baseline + get api("/groups", admin) + + control = ActiveRecord::QueryRecorder.new do + get api("/groups", admin) + end + + create(:group) + + expect do + get api("/groups", admin) + end.not_to exceed_query_limit(control) + end end context "when authenticated as user" do diff --git a/spec/services/system_note_service_spec.rb b/spec/services/system_note_service_spec.rb index ab3aa18cf4e..5b5edc1aa0d 100644 --- a/spec/services/system_note_service_spec.rb +++ b/spec/services/system_note_service_spec.rb @@ -54,10 +54,11 @@ describe SystemNoteService do expect(note_lines[0]).to eq "added #{new_commits.size} commits" end - it 'adds a message line for each commit' do - new_commits.each_with_index do |commit, i| - # Skip the header - expect(HTMLEntities.new.decode(note_lines[i + 1])).to eq "* #{commit.short_id} - #{commit.title}" + it 'adds a message for each commit' do + decoded_note_content = HTMLEntities.new.decode(subject.note) + + new_commits.each do |commit| + expect(decoded_note_content).to include("<li>#{commit.short_id} - #{commit.title}</li>") end end end @@ -69,7 +70,7 @@ describe SystemNoteService do let(:old_commits) { [noteable.commits.last] } it 'includes the existing commit' do - expect(summary_line).to eq "* #{old_commits.first.short_id} - 1 commit from branch `feature`" + expect(summary_line).to start_with("<ul><li>#{old_commits.first.short_id} - 1 commit from branch <code>feature</code>") end end @@ -79,22 +80,16 @@ describe SystemNoteService do context 'with oldrev' do let(:oldrev) { noteable.commits[2].id } - it 'includes a commit range' do - expect(summary_line).to start_with "* #{Commit.truncate_sha(oldrev)}...#{old_commits.last.short_id}" - end - - it 'includes a commit count' do - expect(summary_line).to end_with " - 26 commits from branch `feature`" + it 'includes a commit range and count' do + expect(summary_line) + .to start_with("<ul><li>#{Commit.truncate_sha(oldrev)}...#{old_commits.last.short_id} - 26 commits from branch <code>feature</code>") end end context 'without oldrev' do - it 'includes a commit range' do - expect(summary_line).to start_with "* #{old_commits[0].short_id}..#{old_commits[-1].short_id}" - end - - it 'includes a commit count' do - expect(summary_line).to end_with " - 26 commits from branch `feature`" + it 'includes a commit range and count' do + expect(summary_line) + .to start_with("<ul><li>#{old_commits[0].short_id}..#{old_commits[-1].short_id} - 26 commits from branch <code>feature</code>") end end @@ -104,7 +99,7 @@ describe SystemNoteService do end it 'includes the project namespace' do - expect(summary_line).to end_with "`#{noteable.target_project_namespace}:feature`" + expect(summary_line).to include("<code>#{noteable.target_project_namespace}:feature</code>") end end end @@ -693,7 +688,7 @@ describe SystemNoteService do describe '.new_commit_summary' do it 'escapes HTML titles' do commit = double(title: '<pre>This is a test</pre>', short_id: '12345678') - escaped = '<pre>This is a test</pre>' + escaped = '<pre>This is a test</pre>' expect(described_class.new_commit_summary([commit])).to all(match(/- #{escaped}/)) end diff --git a/vendor/ingress/values.yaml b/vendor/ingress/values.yaml new file mode 100644 index 00000000000..cdb7da77e86 --- /dev/null +++ b/vendor/ingress/values.yaml @@ -0,0 +1,8 @@ +controller: + image: + tag: "0.10.2" + repository: "quay.io/kubernetes-ingress-controller/nginx-ingress-controller" + stats.enabled: true + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "10254" diff --git a/vendor/prometheus/values.yaml b/vendor/prometheus/values.yaml index 5249449c7f8..db967514be7 100644 --- a/vendor/prometheus/values.yaml +++ b/vendor/prometheus/values.yaml @@ -2,7 +2,7 @@ alertmanager: enabled: false kubeStateMetrics: - enabled: false + enabled: true nodeExporter: enabled: false @@ -10,11 +10,15 @@ nodeExporter: pushgateway: enabled: false +server: + image: + tag: v2.1.0 + serverFiles: - alerts: "" - rules: "" + alerts: {} + rules: {} - prometheus.yml: |- + prometheus.yml: rule_files: - /etc/config/rules - /etc/config/alerts @@ -26,92 +30,108 @@ serverFiles: - job_name: kubernetes-cadvisor scheme: https tls_config: - ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: true - bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token" + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: - - role: node - api_server: https://kubernetes.default.svc:443 - tls_config: - ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token" + - role: node relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: - - __meta_kubernetes_node_name - regex: "(.+)" - target_label: __metrics_path__ - replacement: "/api/v1/nodes/${1}/proxy/metrics/cadvisor" + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: + - __meta_kubernetes_node_name + regex: "(.+)" + target_label: __metrics_path__ + replacement: "/api/v1/nodes/${1}/proxy/metrics/cadvisor" metric_relabel_configs: - - source_labels: - - pod_name - target_label: environment - regex: "(.+)-.+-.+" + - source_labels: + - pod_name + target_label: environment + regex: "(.+)-.+-.+" + - job_name: 'kubernetes-service-endpoints' + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: (.+)(?::\d+);(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name - job_name: kubernetes-nodes scheme: https tls_config: - ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: true - bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token" + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: - - role: node - api_server: https://kubernetes.default.svc:443 - tls_config: - ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token" + - role: node relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: - - __meta_kubernetes_node_name - regex: "(.+)" - target_label: __metrics_path__ - replacement: "/api/v1/nodes/${1}/proxy/metrics" + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: + - __meta_kubernetes_node_name + regex: "(.+)" + target_label: __metrics_path__ + replacement: "/api/v1/nodes/${1}/proxy/metrics" metric_relabel_configs: - - source_labels: - - pod_name - target_label: environment - regex: "(.+)-.+-.+" + - source_labels: + - pod_name + target_label: environment + regex: "(.+)-.+-.+" - job_name: kubernetes-pods tls_config: - ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: true - bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token" + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: - - role: pod - api_server: https://kubernetes.default.svc:443 - tls_config: - ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token" + - role: pod relabel_configs: - - source_labels: - - __meta_kubernetes_pod_annotation_prometheus_io_scrape - action: keep - regex: 'true' - - source_labels: - - __meta_kubernetes_pod_annotation_prometheus_io_path - action: replace - target_label: __metrics_path__ - regex: "(.+)" - - source_labels: - - __address__ - - __meta_kubernetes_pod_annotation_prometheus_io_port - action: replace - regex: "([^:]+)(?::[0-9]+)?;([0-9]+)" - replacement: "$1:$2" - target_label: __address__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - source_labels: - - __meta_kubernetes_namespace - action: replace - target_label: kubernetes_namespace - - source_labels: - - __meta_kubernetes_pod_name - action: replace - target_label: kubernetes_pod_name + - source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + action: keep + regex: 'true' + - source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + action: replace + target_label: __metrics_path__ + regex: "(.+)" + - source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + action: replace + regex: "([^:]+)(?::[0-9]+)?;([0-9]+)" + replacement: "$1:$2" + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + action: replace + target_label: kubernetes_namespace + - source_labels: + - __meta_kubernetes_pod_name + action: replace + target_label: kubernetes_pod_name |