diff options
102 files changed, 1839 insertions, 955 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a1552ffb2d..a147bd438b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,6 @@ documentation](doc/development/changelog.md) for instructions on adding your own entry. -## 12.7.2 - -- No changes. - ## 12.7.1 ### Fixed (6 changes) diff --git a/app/assets/javascripts/error_tracking/components/error_tracking_list.vue b/app/assets/javascripts/error_tracking/components/error_tracking_list.vue index 3280ff48129..e57e17c38c1 100644 --- a/app/assets/javascripts/error_tracking/components/error_tracking_list.vue +++ b/app/assets/javascripts/error_tracking/components/error_tracking_list.vue @@ -20,6 +20,8 @@ import TimeAgo from '~/vue_shared/components/time_ago_tooltip.vue'; import { __ } from '~/locale'; import _ from 'underscore'; +export const tableDataClass = 'table-col d-flex d-sm-table-cell'; + export default { FIRST_PAGE: 1, PREV_PAGE: 1, @@ -29,37 +31,37 @@ export default { key: 'error', label: __('Error'), thClass: 'w-60p', - tdClass: 'table-col d-flex d-sm-table-cell px-3', + tdClass: `${tableDataClass} px-3`, }, { key: 'events', label: __('Events'), thClass: 'text-right', - tdClass: 'table-col d-flex d-sm-table-cell', + tdClass: `${tableDataClass}`, }, { key: 'users', label: __('Users'), thClass: 'text-right', - tdClass: 'table-col d-flex d-sm-table-cell', + tdClass: `${tableDataClass}`, }, { key: 'lastSeen', label: __('Last seen'), thClass: '', - tdClass: 'table-col d-flex d-sm-table-cell', + tdClass: `${tableDataClass}`, }, { key: 'ignore', label: '', thClass: 'w-3rem', - tdClass: 'table-col d-flex pl-0 d-sm-table-cell', + tdClass: `${tableDataClass} pl-0`, }, { key: 'resolved', label: '', thClass: 'w-3rem', - tdClass: 'table-col d-flex pl-0 d-sm-table-cell', + tdClass: `${tableDataClass} pl-0`, }, { key: 'details', diff --git a/app/assets/javascripts/jobs/components/sidebar.vue b/app/assets/javascripts/jobs/components/sidebar.vue index 415fa46835b..a61acf2f6eb 100644 --- a/app/assets/javascripts/jobs/components/sidebar.vue +++ b/app/assets/javascripts/jobs/components/sidebar.vue @@ -119,6 +119,7 @@ export default { :class="retryButtonClass" :href="job.retry_path" data-method="post" + data-qa-selector="retry_button" rel="nofollow" >{{ __('Retry') }}</gl-link > diff --git a/app/controllers/projects/git_http_client_controller.rb b/app/controllers/projects/git_http_client_controller.rb deleted file mode 100644 index 3f6e116a62b..00000000000 --- a/app/controllers/projects/git_http_client_controller.rb +++ /dev/null @@ -1,122 +0,0 @@ -# frozen_string_literal: true - -class Projects::GitHttpClientController < Projects::ApplicationController - include ActionController::HttpAuthentication::Basic - include KerberosSpnegoHelper - include Gitlab::Utils::StrongMemoize - - attr_reader :authentication_result, :redirected_path - - delegate :actor, :authentication_abilities, to: :authentication_result, allow_nil: true - delegate :type, to: :authentication_result, allow_nil: true, prefix: :auth_result - - alias_method :user, :actor - alias_method :authenticated_user, :actor - - # Git clients will not know what authenticity token to send along - skip_around_action :set_session_storage - skip_before_action :verify_authenticity_token - skip_before_action :repository - before_action :authenticate_user - - private - - def download_request? - raise NotImplementedError - end - - def upload_request? - raise NotImplementedError - end - - def authenticate_user - @authentication_result = Gitlab::Auth::Result.new - - if allow_basic_auth? && basic_auth_provided? - login, password = user_name_and_password(request) - - if handle_basic_authentication(login, password) - return # Allow access - end - elsif allow_kerberos_spnego_auth? && spnego_provided? - kerberos_user = find_kerberos_user - - if kerberos_user - @authentication_result = Gitlab::Auth::Result.new( - kerberos_user, nil, :kerberos, Gitlab::Auth.full_authentication_abilities) - - send_final_spnego_response - return # Allow access - end - elsif http_download_allowed? - - @authentication_result = Gitlab::Auth::Result.new(nil, project, :none, [:download_code]) - - return # Allow access - end - - send_challenges - render plain: "HTTP Basic: Access denied\n", status: :unauthorized - rescue Gitlab::Auth::MissingPersonalAccessTokenError - render_missing_personal_access_token - end - - def basic_auth_provided? - has_basic_credentials?(request) - end - - def send_challenges - challenges = [] - challenges << 'Basic realm="GitLab"' if allow_basic_auth? - challenges << spnego_challenge if allow_kerberos_spnego_auth? - headers['Www-Authenticate'] = challenges.join("\n") if challenges.any? - end - - def project - parse_repo_path unless defined?(@project) - - @project - end - - def parse_repo_path - @project, @repo_type, @redirected_path = Gitlab::RepoPath.parse("#{params[:namespace_id]}/#{params[:project_id]}") - end - - def render_missing_personal_access_token - render plain: "HTTP Basic: Access denied\n" \ - "You must use a personal access token with 'read_repository' or 'write_repository' scope for Git over HTTP.\n" \ - "You can generate one at #{profile_personal_access_tokens_url}", - status: :unauthorized - end - - def repository - strong_memoize(:repository) do - repo_type.repository_for(project) - end - end - - def repo_type - parse_repo_path unless defined?(@repo_type) - - @repo_type - end - - def handle_basic_authentication(login, password) - @authentication_result = Gitlab::Auth.find_for_git_client( - login, password, project: project, ip: request.ip) - - @authentication_result.success? - end - - def ci? - authentication_result.ci?(project) - end - - def http_download_allowed? - Gitlab::ProtocolAccess.allowed?('http') && - download_request? && - project && Guest.can?(:download_code, project) - end -end - -Projects::GitHttpClientController.prepend_if_ee('EE::Projects::GitHttpClientController') diff --git a/app/controllers/projects/git_http_controller.rb b/app/controllers/projects/git_http_controller.rb deleted file mode 100644 index 236f1b967de..00000000000 --- a/app/controllers/projects/git_http_controller.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: true - -class Projects::GitHttpController < Projects::GitHttpClientController - include WorkhorseRequest - - before_action :access_check - prepend_before_action :deny_head_requests, only: [:info_refs] - - rescue_from Gitlab::GitAccess::UnauthorizedError, with: :render_403_with_exception - rescue_from Gitlab::GitAccess::NotFoundError, with: :render_404_with_exception - rescue_from Gitlab::GitAccess::ProjectCreationError, with: :render_422_with_exception - rescue_from Gitlab::GitAccess::TimeoutError, with: :render_503_with_exception - - # GET /foo/bar.git/info/refs?service=git-upload-pack (git pull) - # GET /foo/bar.git/info/refs?service=git-receive-pack (git push) - def info_refs - log_user_activity if upload_pack? - - render_ok - end - - # POST /foo/bar.git/git-upload-pack (git pull) - def git_upload_pack - enqueue_fetch_statistics_update - - render_ok - end - - # POST /foo/bar.git/git-receive-pack" (git push) - def git_receive_pack - render_ok - end - - private - - def deny_head_requests - head :forbidden if request.head? - end - - def download_request? - upload_pack? - end - - def upload_pack? - git_command == 'git-upload-pack' - end - - def git_command - if action_name == 'info_refs' - params[:service] - else - action_name.dasherize - end - end - - def render_ok - set_workhorse_internal_api_content_type - render json: Gitlab::Workhorse.git_http_ok(repository, repo_type, user, action_name) - end - - def render_403_with_exception(exception) - render plain: exception.message, status: :forbidden - end - - def render_404_with_exception(exception) - render plain: exception.message, status: :not_found - end - - def render_422_with_exception(exception) - render plain: exception.message, status: :unprocessable_entity - end - - def render_503_with_exception(exception) - render plain: exception.message, status: :service_unavailable - end - - def enqueue_fetch_statistics_update - return if Gitlab::Database.read_only? - return if repo_type.wiki? - return unless project&.daily_statistics_enabled? - - ProjectDailyStatisticsWorker.perform_async(project.id) - end - - def access - @access ||= access_klass.new(access_actor, project, 'http', - authentication_abilities: authentication_abilities, - namespace_path: params[:namespace_id], - project_path: project_path, - redirected_path: redirected_path, - auth_result_type: auth_result_type) - end - - def access_actor - return user if user - return :ci if ci? - end - - def access_check - access.check(git_command, Gitlab::GitAccess::ANY) - @project ||= access.project - end - - def access_klass - @access_klass ||= repo_type.access_checker_class - end - - def project_path - @project_path ||= params[:project_id].sub(/\.git$/, '') - end - - def log_user_activity - Users::ActivityService.new(user).execute - end -end - -Projects::GitHttpController.prepend_if_ee('EE::Projects::GitHttpController') diff --git a/app/controllers/projects/lfs_api_controller.rb b/app/controllers/projects/lfs_api_controller.rb deleted file mode 100644 index 1273c55b83a..00000000000 --- a/app/controllers/projects/lfs_api_controller.rb +++ /dev/null @@ -1,140 +0,0 @@ -# frozen_string_literal: true - -class Projects::LfsApiController < Projects::GitHttpClientController - include LfsRequest - include Gitlab::Utils::StrongMemoize - - LFS_TRANSFER_CONTENT_TYPE = 'application/octet-stream' - - skip_before_action :lfs_check_access!, only: [:deprecated] - before_action :lfs_check_batch_operation!, only: [:batch] - - def batch - unless objects.present? - render_lfs_not_found - return - end - - if download_request? - render json: { objects: download_objects! } - elsif upload_request? - render json: { objects: upload_objects! } - else - raise "Never reached" - end - end - - def deprecated - render( - json: { - message: _('Server supports batch API only, please update your Git LFS client to version 1.0.1 and up.'), - documentation_url: "#{Gitlab.config.gitlab.url}/help" - }, - status: :not_implemented - ) - end - - private - - def download_request? - params[:operation] == 'download' - end - - def upload_request? - params[:operation] == 'upload' - end - - # rubocop: disable CodeReuse/ActiveRecord - def existing_oids - @existing_oids ||= begin - project.all_lfs_objects.where(oid: objects.map { |o| o['oid'].to_s }).pluck(:oid) - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def download_objects! - objects.each do |object| - if existing_oids.include?(object[:oid]) - object[:actions] = download_actions(object) - - if Guest.can?(:download_code, project) - object[:authenticated] = true - end - else - object[:error] = { - code: 404, - message: _("Object does not exist on the server or you don't have permissions to access it") - } - end - end - objects - end - - def upload_objects! - objects.each do |object| - object[:actions] = upload_actions(object) unless existing_oids.include?(object[:oid]) - end - objects - end - - def download_actions(object) - { - download: { - href: "#{project.http_url_to_repo}/gitlab-lfs/objects/#{object[:oid]}", - header: { - Authorization: authorization_header - }.compact - } - } - end - - def upload_actions(object) - { - upload: { - href: "#{project.http_url_to_repo}/gitlab-lfs/objects/#{object[:oid]}/#{object[:size]}", - header: { - Authorization: authorization_header, - # git-lfs v2.5.0 sets the Content-Type based on the uploaded file. This - # ensures that Workhorse can intercept the request. - 'Content-Type': LFS_TRANSFER_CONTENT_TYPE - }.compact - } - } - end - - def lfs_check_batch_operation! - if batch_operation_disallowed? - render( - json: { - message: lfs_read_only_message - }, - content_type: LfsRequest::CONTENT_TYPE, - status: :forbidden - ) - end - end - - # Overridden in EE - def batch_operation_disallowed? - upload_request? && Gitlab::Database.read_only? - end - - # Overridden in EE - def lfs_read_only_message - _('You cannot write to this read-only GitLab instance.') - end - - def authorization_header - strong_memoize(:authorization_header) do - lfs_auth_header || request.headers['Authorization'] - end - end - - def lfs_auth_header - return unless user.is_a?(User) - - Gitlab::LfsToken.new(user).basic_encoding - end -end - -Projects::LfsApiController.prepend_if_ee('EE::Projects::LfsApiController') diff --git a/app/controllers/projects/lfs_locks_api_controller.rb b/app/controllers/projects/lfs_locks_api_controller.rb deleted file mode 100644 index 6aacb9d9a56..00000000000 --- a/app/controllers/projects/lfs_locks_api_controller.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -class Projects::LfsLocksApiController < Projects::GitHttpClientController - include LfsRequest - - def create - @result = Lfs::LockFileService.new(project, user, lfs_params).execute - - render_json(@result[:lock]) - end - - def unlock - @result = Lfs::UnlockFileService.new(project, user, lfs_params).execute - - render_json(@result[:lock]) - end - - def index - @result = Lfs::LocksFinderService.new(project, user, lfs_params).execute - - render_json(@result[:locks]) - end - - def verify - @result = Lfs::LocksFinderService.new(project, user, {}).execute - - ours, theirs = split_by_owner(@result[:locks]) - - render_json({ ours: ours, theirs: theirs }, false) - end - - private - - def render_json(data, process = true) - render json: build_payload(data, process), - content_type: LfsRequest::CONTENT_TYPE, - status: @result[:http_status] - end - - def build_payload(data, process) - data = LfsFileLockSerializer.new.represent(data) if process - - return data if @result[:status] == :success - - # When the locking failed due to an existent Lock, the existent record - # is returned in `@result[:lock]` - error_payload(@result[:message], @result[:lock] ? data : {}) - end - - def error_payload(message, custom_attrs = {}) - custom_attrs.merge({ - message: message, - documentation_url: help_url - }) - end - - def split_by_owner(locks) - groups = locks.partition { |lock| lock.user_id == user.id } - - groups.map! do |records| - LfsFileLockSerializer.new.represent(records, root: false) - end - end - - def download_request? - params[:action] == 'index' - end - - def upload_request? - %w(create unlock verify).include?(params[:action]) - end - - def lfs_params - params.permit(:id, :path, :force) - end -end diff --git a/app/controllers/projects/lfs_storage_controller.rb b/app/controllers/projects/lfs_storage_controller.rb deleted file mode 100644 index 013e01b82aa..00000000000 --- a/app/controllers/projects/lfs_storage_controller.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -class Projects::LfsStorageController < Projects::GitHttpClientController - include LfsRequest - include WorkhorseRequest - include SendFileUpload - - skip_before_action :verify_workhorse_api!, only: :download - - def download - lfs_object = LfsObject.find_by_oid(oid) - unless lfs_object && lfs_object.file.exists? - render_lfs_not_found - return - end - - send_upload(lfs_object.file, send_params: { content_type: "application/octet-stream" }) - end - - def upload_authorize - set_workhorse_internal_api_content_type - - authorized = LfsObjectUploader.workhorse_authorize(has_length: true) - authorized.merge!(LfsOid: oid, LfsSize: size) - - render json: authorized - end - - def upload_finalize - if store_file!(oid, size) - head 200 - else - render plain: 'Unprocessable entity', status: :unprocessable_entity - end - rescue ActiveRecord::RecordInvalid - render_lfs_forbidden - rescue UploadedFile::InvalidPathError - render_lfs_forbidden - rescue ObjectStorage::RemoteStoreError - render_lfs_forbidden - end - - private - - def download_request? - action_name == 'download' - end - - def upload_request? - %w[upload_authorize upload_finalize].include? action_name - end - - def oid - params[:oid].to_s - end - - def size - params[:size].to_i - end - - # rubocop: disable CodeReuse/ActiveRecord - def store_file!(oid, size) - object = LfsObject.find_by(oid: oid, size: size) - unless object&.file&.exists? - object = create_file!(oid, size) - end - - return unless object - - link_to_project!(object) - end - # rubocop: enable CodeReuse/ActiveRecord - - def create_file!(oid, size) - uploaded_file = UploadedFile.from_params( - params, :file, LfsObjectUploader.workhorse_local_upload_path) - return unless uploaded_file - - LfsObject.create!(oid: oid, size: size, file: uploaded_file) - end - - # rubocop: disable CodeReuse/ActiveRecord - def link_to_project!(object) - if object && !object.projects.exists?(storage_project.id) - object.lfs_objects_projects.create!(project: storage_project) - end - end - # rubocop: enable CodeReuse/ActiveRecord -end diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index bf05defbc2e..f4f2a16b82b 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -387,6 +387,7 @@ class ProjectsController < Projects::ApplicationController :merge_method, :initialize_with_readme, :autoclose_referenced_issues, + :suggestion_commit_message, project_feature_attributes: %i[ builds_access_level diff --git a/app/controllers/repositories/application_controller.rb b/app/controllers/repositories/application_controller.rb new file mode 100644 index 00000000000..528cc310038 --- /dev/null +++ b/app/controllers/repositories/application_controller.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Repositories + class ApplicationController < ::ApplicationController + skip_before_action :authenticate_user! + end +end diff --git a/app/controllers/repositories/git_http_client_controller.rb b/app/controllers/repositories/git_http_client_controller.rb new file mode 100644 index 00000000000..76eb7c67205 --- /dev/null +++ b/app/controllers/repositories/git_http_client_controller.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +module Repositories + class GitHttpClientController < Repositories::ApplicationController + include ActionController::HttpAuthentication::Basic + include KerberosSpnegoHelper + include Gitlab::Utils::StrongMemoize + + attr_reader :authentication_result, :redirected_path + + delegate :actor, :authentication_abilities, to: :authentication_result, allow_nil: true + delegate :type, to: :authentication_result, allow_nil: true, prefix: :auth_result + + alias_method :user, :actor + alias_method :authenticated_user, :actor + + # Git clients will not know what authenticity token to send along + skip_around_action :set_session_storage + skip_before_action :verify_authenticity_token + + before_action :parse_repo_path + before_action :authenticate_user + + private + + def download_request? + raise NotImplementedError + end + + def upload_request? + raise NotImplementedError + end + + def authenticate_user + @authentication_result = Gitlab::Auth::Result.new + + if allow_basic_auth? && basic_auth_provided? + login, password = user_name_and_password(request) + + if handle_basic_authentication(login, password) + return # Allow access + end + elsif allow_kerberos_spnego_auth? && spnego_provided? + kerberos_user = find_kerberos_user + + if kerberos_user + @authentication_result = Gitlab::Auth::Result.new( + kerberos_user, nil, :kerberos, Gitlab::Auth.full_authentication_abilities) + + send_final_spnego_response + return # Allow access + end + elsif http_download_allowed? + + @authentication_result = Gitlab::Auth::Result.new(nil, project, :none, [:download_code]) + + return # Allow access + end + + send_challenges + render plain: "HTTP Basic: Access denied\n", status: :unauthorized + rescue Gitlab::Auth::MissingPersonalAccessTokenError + render_missing_personal_access_token + end + + def basic_auth_provided? + has_basic_credentials?(request) + end + + def send_challenges + challenges = [] + challenges << 'Basic realm="GitLab"' if allow_basic_auth? + challenges << spnego_challenge if allow_kerberos_spnego_auth? + headers['Www-Authenticate'] = challenges.join("\n") if challenges.any? + end + + def project + parse_repo_path unless defined?(@project) + + @project + end + + def parse_repo_path + @project, @repo_type, @redirected_path = Gitlab::RepoPath.parse("#{params[:namespace_id]}/#{params[:repository_id]}") + end + + def render_missing_personal_access_token + render plain: "HTTP Basic: Access denied\n" \ + "You must use a personal access token with 'read_repository' or 'write_repository' scope for Git over HTTP.\n" \ + "You can generate one at #{profile_personal_access_tokens_url}", + status: :unauthorized + end + + def repository + strong_memoize(:repository) do + repo_type.repository_for(project) + end + end + + def repo_type + parse_repo_path unless defined?(@repo_type) + + @repo_type + end + + def handle_basic_authentication(login, password) + @authentication_result = Gitlab::Auth.find_for_git_client( + login, password, project: project, ip: request.ip) + + @authentication_result.success? + end + + def ci? + authentication_result.ci?(project) + end + + def http_download_allowed? + Gitlab::ProtocolAccess.allowed?('http') && + download_request? && + project && Guest.can?(:download_code, project) + end + end +end + +Repositories::GitHttpClientController.prepend_if_ee('EE::Repositories::GitHttpClientController') diff --git a/app/controllers/repositories/git_http_controller.rb b/app/controllers/repositories/git_http_controller.rb new file mode 100644 index 00000000000..82431ae286a --- /dev/null +++ b/app/controllers/repositories/git_http_controller.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module Repositories + class GitHttpController < Repositories::GitHttpClientController + include WorkhorseRequest + + before_action :access_check + prepend_before_action :deny_head_requests, only: [:info_refs] + + rescue_from Gitlab::GitAccess::UnauthorizedError, with: :render_403_with_exception + rescue_from Gitlab::GitAccess::NotFoundError, with: :render_404_with_exception + rescue_from Gitlab::GitAccess::ProjectCreationError, with: :render_422_with_exception + rescue_from Gitlab::GitAccess::TimeoutError, with: :render_503_with_exception + + # GET /foo/bar.git/info/refs?service=git-upload-pack (git pull) + # GET /foo/bar.git/info/refs?service=git-receive-pack (git push) + def info_refs + log_user_activity if upload_pack? + + render_ok + end + + # POST /foo/bar.git/git-upload-pack (git pull) + def git_upload_pack + enqueue_fetch_statistics_update + + render_ok + end + + # POST /foo/bar.git/git-receive-pack" (git push) + def git_receive_pack + render_ok + end + + private + + def deny_head_requests + head :forbidden if request.head? + end + + def download_request? + upload_pack? + end + + def upload_pack? + git_command == 'git-upload-pack' + end + + def git_command + if action_name == 'info_refs' + params[:service] + else + action_name.dasherize + end + end + + def render_ok + set_workhorse_internal_api_content_type + render json: Gitlab::Workhorse.git_http_ok(repository, repo_type, user, action_name) + end + + def render_403_with_exception(exception) + render plain: exception.message, status: :forbidden + end + + def render_404_with_exception(exception) + render plain: exception.message, status: :not_found + end + + def render_422_with_exception(exception) + render plain: exception.message, status: :unprocessable_entity + end + + def render_503_with_exception(exception) + render plain: exception.message, status: :service_unavailable + end + + def enqueue_fetch_statistics_update + return if Gitlab::Database.read_only? + return unless repo_type.project? + return unless project&.daily_statistics_enabled? + + ProjectDailyStatisticsWorker.perform_async(project.id) + end + + def access + @access ||= access_klass.new(access_actor, project, 'http', + authentication_abilities: authentication_abilities, + namespace_path: params[:namespace_id], + project_path: project_path, + redirected_path: redirected_path, + auth_result_type: auth_result_type) + end + + def access_actor + return user if user + return :ci if ci? + end + + def access_check + access.check(git_command, Gitlab::GitAccess::ANY) + @project ||= access.project + end + + def access_klass + @access_klass ||= repo_type.access_checker_class + end + + def project_path + @project_path ||= params[:repository_id].sub(/\.git$/, '') + end + + def log_user_activity + Users::ActivityService.new(user).execute + end + end +end + +Repositories::GitHttpController.prepend_if_ee('EE::Repositories::GitHttpController') diff --git a/app/controllers/repositories/lfs_api_controller.rb b/app/controllers/repositories/lfs_api_controller.rb new file mode 100644 index 00000000000..b1e0d1848d7 --- /dev/null +++ b/app/controllers/repositories/lfs_api_controller.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +module Repositories + class LfsApiController < Repositories::GitHttpClientController + include LfsRequest + include Gitlab::Utils::StrongMemoize + + LFS_TRANSFER_CONTENT_TYPE = 'application/octet-stream' + + skip_before_action :lfs_check_access!, only: [:deprecated] + before_action :lfs_check_batch_operation!, only: [:batch] + + def batch + unless objects.present? + render_lfs_not_found + return + end + + if download_request? + render json: { objects: download_objects! } + elsif upload_request? + render json: { objects: upload_objects! } + else + raise "Never reached" + end + end + + def deprecated + render( + json: { + message: _('Server supports batch API only, please update your Git LFS client to version 1.0.1 and up.'), + documentation_url: "#{Gitlab.config.gitlab.url}/help" + }, + status: :not_implemented + ) + end + + private + + def download_request? + params[:operation] == 'download' + end + + def upload_request? + params[:operation] == 'upload' + end + + # rubocop: disable CodeReuse/ActiveRecord + def existing_oids + @existing_oids ||= begin + project.all_lfs_objects.where(oid: objects.map { |o| o['oid'].to_s }).pluck(:oid) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def download_objects! + objects.each do |object| + if existing_oids.include?(object[:oid]) + object[:actions] = download_actions(object) + + if Guest.can?(:download_code, project) + object[:authenticated] = true + end + else + object[:error] = { + code: 404, + message: _("Object does not exist on the server or you don't have permissions to access it") + } + end + end + objects + end + + def upload_objects! + objects.each do |object| + object[:actions] = upload_actions(object) unless existing_oids.include?(object[:oid]) + end + objects + end + + def download_actions(object) + { + download: { + href: "#{project.http_url_to_repo}/gitlab-lfs/objects/#{object[:oid]}", + header: { + Authorization: authorization_header + }.compact + } + } + end + + def upload_actions(object) + { + upload: { + href: "#{project.http_url_to_repo}/gitlab-lfs/objects/#{object[:oid]}/#{object[:size]}", + header: { + Authorization: authorization_header, + # git-lfs v2.5.0 sets the Content-Type based on the uploaded file. This + # ensures that Workhorse can intercept the request. + 'Content-Type': LFS_TRANSFER_CONTENT_TYPE + }.compact + } + } + end + + def lfs_check_batch_operation! + if batch_operation_disallowed? + render( + json: { + message: lfs_read_only_message + }, + content_type: LfsRequest::CONTENT_TYPE, + status: :forbidden + ) + end + end + + # Overridden in EE + def batch_operation_disallowed? + upload_request? && Gitlab::Database.read_only? + end + + # Overridden in EE + def lfs_read_only_message + _('You cannot write to this read-only GitLab instance.') + end + + def authorization_header + strong_memoize(:authorization_header) do + lfs_auth_header || request.headers['Authorization'] + end + end + + def lfs_auth_header + return unless user.is_a?(User) + + Gitlab::LfsToken.new(user).basic_encoding + end + end +end + +Repositories::LfsApiController.prepend_if_ee('EE::Repositories::LfsApiController') diff --git a/app/controllers/repositories/lfs_locks_api_controller.rb b/app/controllers/repositories/lfs_locks_api_controller.rb new file mode 100644 index 00000000000..19fc09ad4de --- /dev/null +++ b/app/controllers/repositories/lfs_locks_api_controller.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Repositories + class LfsLocksApiController < Repositories::GitHttpClientController + include LfsRequest + + def create + @result = Lfs::LockFileService.new(project, user, lfs_params).execute + + render_json(@result[:lock]) + end + + def unlock + @result = Lfs::UnlockFileService.new(project, user, lfs_params).execute + + render_json(@result[:lock]) + end + + def index + @result = Lfs::LocksFinderService.new(project, user, lfs_params).execute + + render_json(@result[:locks]) + end + + def verify + @result = Lfs::LocksFinderService.new(project, user, {}).execute + + ours, theirs = split_by_owner(@result[:locks]) + + render_json({ ours: ours, theirs: theirs }, false) + end + + private + + def render_json(data, process = true) + render json: build_payload(data, process), + content_type: LfsRequest::CONTENT_TYPE, + status: @result[:http_status] + end + + def build_payload(data, process) + data = LfsFileLockSerializer.new.represent(data) if process + + return data if @result[:status] == :success + + # When the locking failed due to an existent Lock, the existent record + # is returned in `@result[:lock]` + error_payload(@result[:message], @result[:lock] ? data : {}) + end + + def error_payload(message, custom_attrs = {}) + custom_attrs.merge({ + message: message, + documentation_url: help_url + }) + end + + def split_by_owner(locks) + groups = locks.partition { |lock| lock.user_id == user.id } + + groups.map! do |records| + LfsFileLockSerializer.new.represent(records, root: false) + end + end + + def download_request? + params[:action] == 'index' + end + + def upload_request? + %w(create unlock verify).include?(params[:action]) + end + + def lfs_params + params.permit(:id, :path, :force) + end + end +end diff --git a/app/controllers/repositories/lfs_storage_controller.rb b/app/controllers/repositories/lfs_storage_controller.rb new file mode 100644 index 00000000000..58f496e16d3 --- /dev/null +++ b/app/controllers/repositories/lfs_storage_controller.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Repositories + class LfsStorageController < Repositories::GitHttpClientController + include LfsRequest + include WorkhorseRequest + include SendFileUpload + + skip_before_action :verify_workhorse_api!, only: :download + + def download + lfs_object = LfsObject.find_by_oid(oid) + unless lfs_object && lfs_object.file.exists? + render_lfs_not_found + return + end + + send_upload(lfs_object.file, send_params: { content_type: "application/octet-stream" }) + end + + def upload_authorize + set_workhorse_internal_api_content_type + + authorized = LfsObjectUploader.workhorse_authorize(has_length: true) + authorized.merge!(LfsOid: oid, LfsSize: size) + + render json: authorized + end + + def upload_finalize + if store_file!(oid, size) + head 200 + else + render plain: 'Unprocessable entity', status: :unprocessable_entity + end + rescue ActiveRecord::RecordInvalid + render_lfs_forbidden + rescue UploadedFile::InvalidPathError + render_lfs_forbidden + rescue ObjectStorage::RemoteStoreError + render_lfs_forbidden + end + + private + + def download_request? + action_name == 'download' + end + + def upload_request? + %w[upload_authorize upload_finalize].include? action_name + end + + def oid + params[:oid].to_s + end + + def size + params[:size].to_i + end + + # rubocop: disable CodeReuse/ActiveRecord + def store_file!(oid, size) + object = LfsObject.find_by(oid: oid, size: size) + unless object&.file&.exists? + object = create_file!(oid, size) + end + + return unless object + + link_to_project!(object) + end + # rubocop: enable CodeReuse/ActiveRecord + + def create_file!(oid, size) + uploaded_file = UploadedFile.from_params( + params, :file, LfsObjectUploader.workhorse_local_upload_path) + return unless uploaded_file + + LfsObject.create!(oid: oid, size: size, file: uploaded_file) + end + + # rubocop: disable CodeReuse/ActiveRecord + def link_to_project!(object) + if object && !object.projects.exists?(storage_project.id) + object.lfs_objects_projects.create!(project: storage_project) + end + end + # rubocop: enable CodeReuse/ActiveRecord + end +end diff --git a/app/graphql/types/error_tracking/sentry_detailed_error_type.rb b/app/graphql/types/error_tracking/sentry_detailed_error_type.rb index af6d8818d90..e3ccf9e61c8 100644 --- a/app/graphql/types/error_tracking/sentry_detailed_error_type.rb +++ b/app/graphql/types/error_tracking/sentry_detailed_error_type.rb @@ -11,77 +11,87 @@ module Types field :id, GraphQL::ID_TYPE, null: false, - description: "ID (global ID) of the error" + description: 'ID (global ID) of the error' field :sentry_id, GraphQL::STRING_TYPE, method: :id, null: false, - description: "ID (Sentry ID) of the error" + description: 'ID (Sentry ID) of the error' field :title, GraphQL::STRING_TYPE, null: false, - description: "Title of the error" + description: 'Title of the error' field :type, GraphQL::STRING_TYPE, null: false, - description: "Type of the error" + description: 'Type of the error' field :user_count, GraphQL::INT_TYPE, null: false, - description: "Count of users affected by the error" + description: 'Count of users affected by the error' field :count, GraphQL::INT_TYPE, null: false, - description: "Count of occurrences" + description: 'Count of occurrences' field :first_seen, Types::TimeType, null: false, - description: "Timestamp when the error was first seen" + description: 'Timestamp when the error was first seen' field :last_seen, Types::TimeType, null: false, - description: "Timestamp when the error was last seen" + description: 'Timestamp when the error was last seen' field :message, GraphQL::STRING_TYPE, null: true, - description: "Sentry metadata message of the error" + description: 'Sentry metadata message of the error' field :culprit, GraphQL::STRING_TYPE, null: false, - description: "Culprit of the error" + description: 'Culprit of the error' + field :external_base_url, GraphQL::STRING_TYPE, + null: false, + description: 'External Base URL of the Sentry Instance' field :external_url, GraphQL::STRING_TYPE, null: false, - description: "External URL of the error" + description: 'External URL of the error' field :sentry_project_id, GraphQL::ID_TYPE, method: :project_id, null: false, - description: "ID of the project (Sentry project)" + description: 'ID of the project (Sentry project)' field :sentry_project_name, GraphQL::STRING_TYPE, method: :project_name, null: false, - description: "Name of the project affected by the error" + description: 'Name of the project affected by the error' field :sentry_project_slug, GraphQL::STRING_TYPE, method: :project_slug, null: false, - description: "Slug of the project affected by the error" + description: 'Slug of the project affected by the error' field :short_id, GraphQL::STRING_TYPE, null: false, - description: "Short ID (Sentry ID) of the error" + description: 'Short ID (Sentry ID) of the error' field :status, Types::ErrorTracking::SentryErrorStatusEnum, null: false, - description: "Status of the error" + description: 'Status of the error' field :frequency, [Types::ErrorTracking::SentryErrorFrequencyType], null: false, - description: "Last 24hr stats of the error" + description: 'Last 24hr stats of the error' field :first_release_last_commit, GraphQL::STRING_TYPE, null: true, - description: "Commit the error was first seen" + description: 'Commit the error was first seen' field :last_release_last_commit, GraphQL::STRING_TYPE, null: true, - description: "Commit the error was last seen" + description: 'Commit the error was last seen' field :first_release_short_version, GraphQL::STRING_TYPE, null: true, - description: "Release version the error was first seen" + description: 'Release version the error was first seen' field :last_release_short_version, GraphQL::STRING_TYPE, null: true, - description: "Release version the error was last seen" + description: 'Release version the error was last seen' field :gitlab_commit, GraphQL::STRING_TYPE, null: true, - description: "GitLab commit SHA attributed to the Error based on the release version" + description: 'GitLab commit SHA attributed to the Error based on the release version' field :gitlab_commit_path, GraphQL::STRING_TYPE, null: true, - description: "Path to the GitLab page for the GitLab commit attributed to the error" + description: 'Path to the GitLab page for the GitLab commit attributed to the error' + field :gitlab_issue_path, GraphQL::STRING_TYPE, + method: :gitlab_issue, + null: true, + description: 'URL of GitLab Issue' + field :tags, Types::ErrorTracking::SentryErrorTagsType, + null: false, + description: 'Tags associated with the Sentry Error' def first_seen DateTime.parse(object.first_seen) diff --git a/app/graphql/types/error_tracking/sentry_error_tags_type.rb b/app/graphql/types/error_tracking/sentry_error_tags_type.rb new file mode 100644 index 00000000000..e6d96571561 --- /dev/null +++ b/app/graphql/types/error_tracking/sentry_error_tags_type.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Types + module ErrorTracking + # rubocop: disable Graphql/AuthorizeTypes + class SentryErrorTagsType < ::Types::BaseObject + graphql_name 'SentryErrorTags' + description 'State of a Sentry error' + + field :level, GraphQL::STRING_TYPE, + null: true, + description: "Severity level of the Sentry Error" + field :logger, GraphQL::STRING_TYPE, + null: true, + description: "Logger of the Sentry Error" + end + # rubocop: enable Graphql/AuthorizeTypes + end +end diff --git a/app/helpers/blob_helper.rb b/app/helpers/blob_helper.rb index c9fb28d0299..7c0f4da355d 100644 --- a/app/helpers/blob_helper.rb +++ b/app/helpers/blob_helper.rb @@ -47,7 +47,7 @@ module BlobHelper def edit_blob_button(project = @project, ref = @ref, path = @path, options = {}) return unless blob = readable_blob(options, path, project, ref) - common_classes = "btn btn-primary js-edit-blob #{options[:extra_class]}" + common_classes = "btn btn-primary js-edit-blob ml-2 #{options[:extra_class]}" edit_button_tag(blob, common_classes, @@ -62,7 +62,7 @@ module BlobHelper return unless blob = readable_blob(options, path, project, ref) edit_button_tag(blob, - 'btn btn-inverted btn-primary ide-edit-button', + 'btn btn-inverted btn-primary ide-edit-button ml-2', _('Web IDE'), ide_edit_path(project, ref, path, options), project, diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb index 63f216c7af5..bf189d0f517 100644 --- a/app/models/clusters/applications/ingress.rb +++ b/app/models/clusters/applications/ingress.rb @@ -4,6 +4,7 @@ module Clusters module Applications class Ingress < ApplicationRecord VERSION = '1.22.1' + MODSECURITY_LOG_CONTAINER_NAME = 'modsecurity-log' self.table_name = 'clusters_applications_ingress' @@ -85,7 +86,7 @@ module Clusters }, "extraContainers" => [ { - "name" => "modsecurity-log", + "name" => MODSECURITY_LOG_CONTAINER_NAME, "image" => "busybox", "args" => [ "/bin/sh", diff --git a/app/models/project.rb b/app/models/project.rb index cd191589351..c0103190892 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1938,6 +1938,8 @@ class Project < ApplicationRecord .append(key: 'GITLAB_CI', value: 'true') .append(key: 'CI_SERVER_URL', value: Gitlab.config.gitlab.url) .append(key: 'CI_SERVER_HOST', value: Gitlab.config.gitlab.host) + .append(key: 'CI_SERVER_PORT', value: Gitlab.config.gitlab.port.to_s) + .append(key: 'CI_SERVER_PROTOCOL', value: Gitlab.config.gitlab.protocol) .append(key: 'CI_SERVER_NAME', value: 'GitLab') .append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION) .append(key: 'CI_SERVER_VERSION_MAJOR', value: Gitlab.version_info.major.to_s) diff --git a/app/views/projects/blob/_header.html.haml b/app/views/projects/blob/_header.html.haml index 77245114772..76a9d3df5d7 100644 --- a/app/views/projects/blob/_header.html.haml +++ b/app/views/projects/blob/_header.html.haml @@ -2,18 +2,16 @@ .js-file-title.file-title-flex-parent = render 'projects/blob/header_content', blob: blob - .file-actions + .file-actions< = render 'projects/blob/viewer_switcher', blob: blob unless blame - - .btn-group{ role: "group" }< - = edit_blob_button - = ide_edit_button - .btn-group{ role: "group" }< + = edit_blob_button + = ide_edit_button + .btn-group.ml-2{ role: "group" }> = render_if_exists 'projects/blob/header_file_locks_link' - if current_user = replace_blob_link = delete_blob_link - .btn-group{ role: "group" }< + .btn-group.ml-2{ role: "group" } = copy_blob_source_button(blob) unless blame = open_raw_blob_button(blob) = download_blob_button(blob) diff --git a/app/views/projects/blob/_viewer_switcher.html.haml b/app/views/projects/blob/_viewer_switcher.html.haml index 6a521069418..5e0d70b2ca9 100644 --- a/app/views/projects/blob/_viewer_switcher.html.haml +++ b/app/views/projects/blob/_viewer_switcher.html.haml @@ -2,7 +2,7 @@ - simple_viewer = blob.simple_viewer - rich_viewer = blob.rich_viewer - .btn-group.js-blob-viewer-switcher{ role: "group" } + .btn-group.js-blob-viewer-switcher.ml-2{ role: "group" }> - simple_label = "Display #{simple_viewer.switcher_title}" %button.btn.btn-default.btn-sm.js-blob-viewer-switch-btn.has-tooltip{ 'aria-label' => simple_label, title: simple_label, data: { viewer: 'simple', container: 'body' } }> = icon(simple_viewer.switcher_icon) diff --git a/app/views/shared/members/_member.html.haml b/app/views/shared/members/_member.html.haml index d5c1a1bee6d..d74030c566f 100644 --- a/app/views/shared/members/_member.html.haml +++ b/app/views/shared/members/_member.html.haml @@ -22,6 +22,8 @@ - if user == current_user %span.badge.badge-success.prepend-left-5= _("It's you") + = render_if_exists 'shared/members/ee/license_badge', user: user, group: @group + - if user.blocked? %label.badge.badge-danger %strong= _("Blocked") diff --git a/app/workers/authorized_projects_worker.rb b/app/workers/authorized_projects_worker.rb index 9492cfe217c..1ab2fd6023f 100644 --- a/app/workers/authorized_projects_worker.rb +++ b/app/workers/authorized_projects_worker.rb @@ -6,6 +6,7 @@ class AuthorizedProjectsWorker feature_category :authentication_and_authorization latency_sensitive_worker! + weight 2 # This is a workaround for a Ruby 2.3.7 bug. rspec-mocks cannot restore the # visibility of prepended modules. See https://github.com/rspec/rspec-mocks/issues/1231 diff --git a/app/workers/chat_notification_worker.rb b/app/workers/chat_notification_worker.rb index 6162dcf9d38..f23c787559f 100644 --- a/app/workers/chat_notification_worker.rb +++ b/app/workers/chat_notification_worker.rb @@ -8,6 +8,8 @@ class ChatNotificationWorker sidekiq_options retry: false feature_category :chatops latency_sensitive_worker! + weight 2 + # TODO: break this into multiple jobs # as the `responder` uses external dependencies # See https://gitlab.com/gitlab-com/gl-infra/scalability/issues/34 diff --git a/app/workers/concerns/self_monitoring_project_worker.rb b/app/workers/concerns/self_monitoring_project_worker.rb index 44dd6866fad..1796e2441f2 100644 --- a/app/workers/concerns/self_monitoring_project_worker.rb +++ b/app/workers/concerns/self_monitoring_project_worker.rb @@ -9,6 +9,7 @@ module SelfMonitoringProjectWorker # Other Functionality. Metrics seems to be the closest feature_category for # this worker. feature_category :metrics + weight 2 end LEASE_TIMEOUT = 15.minutes.to_i diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb index 506215ca9ed..266c8021761 100644 --- a/app/workers/concerns/worker_attributes.rb +++ b/app/workers/concerns/worker_attributes.rb @@ -7,6 +7,24 @@ module WorkerAttributes # `worker_resource_boundary` attribute VALID_RESOURCE_BOUNDARIES = [:memory, :cpu, :unknown].freeze + NAMESPACE_WEIGHTS = { + auto_devops: 2, + auto_merge: 3, + chaos: 2, + deployment: 3, + mail_scheduler: 2, + notifications: 2, + pipeline_cache: 3, + pipeline_creation: 4, + pipeline_default: 3, + pipeline_hooks: 2, + pipeline_processing: 5, + + # EE-specific + epics: 2, + incident_management: 2 + }.stringify_keys.freeze + class_methods do def feature_category(value) raise "Invalid category. Use `feature_category_not_owned!` to mark a worker as not owned" if value == :not_owned @@ -70,6 +88,16 @@ module WorkerAttributes worker_attributes[:resource_boundary] || :unknown end + def weight(value) + worker_attributes[:weight] = value + end + + def get_weight + worker_attributes[:weight] || + NAMESPACE_WEIGHTS[queue_namespace] || + 1 + end + protected # Returns a worker attribute declared on this class or its parent class. diff --git a/app/workers/create_evidence_worker.rb b/app/workers/create_evidence_worker.rb index 027dbd2f101..e6fbf59d702 100644 --- a/app/workers/create_evidence_worker.rb +++ b/app/workers/create_evidence_worker.rb @@ -4,6 +4,7 @@ class CreateEvidenceWorker include ApplicationWorker feature_category :release_governance + weight 2 def perform(release_id) release = Release.find_by_id(release_id) diff --git a/app/workers/create_gpg_signature_worker.rb b/app/workers/create_gpg_signature_worker.rb index fc36a2adccd..2043c3c8e77 100644 --- a/app/workers/create_gpg_signature_worker.rb +++ b/app/workers/create_gpg_signature_worker.rb @@ -4,6 +4,7 @@ class CreateGpgSignatureWorker include ApplicationWorker feature_category :source_code_management + weight 2 # rubocop: disable CodeReuse/ActiveRecord def perform(commit_shas, project_id) diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb index b56bf4ed833..c2b1e642604 100644 --- a/app/workers/email_receiver_worker.rb +++ b/app/workers/email_receiver_worker.rb @@ -5,6 +5,7 @@ class EmailReceiverWorker feature_category :issue_tracking latency_sensitive_worker! + weight 2 def perform(raw) return unless Gitlab::IncomingEmail.enabled? diff --git a/app/workers/emails_on_push_worker.rb b/app/workers/emails_on_push_worker.rb index f523f5953e1..be66e2b1188 100644 --- a/app/workers/emails_on_push_worker.rb +++ b/app/workers/emails_on_push_worker.rb @@ -8,6 +8,7 @@ class EmailsOnPushWorker feature_category :source_code_management latency_sensitive_worker! worker_resource_boundary :cpu + weight 2 def perform(project_id, recipients, push_data, options = {}) options.symbolize_keys! diff --git a/app/workers/gitlab_shell_worker.rb b/app/workers/gitlab_shell_worker.rb index 57e64570c09..bd2225e6d7c 100644 --- a/app/workers/gitlab_shell_worker.rb +++ b/app/workers/gitlab_shell_worker.rb @@ -6,6 +6,7 @@ class GitlabShellWorker feature_category :source_code_management latency_sensitive_worker! + weight 2 def perform(action, *arg) Gitlab::GitalyClient::NamespaceService.allow do diff --git a/app/workers/import_issues_csv_worker.rb b/app/workers/import_issues_csv_worker.rb index d2733dc5f56..7c5584146ca 100644 --- a/app/workers/import_issues_csv_worker.rb +++ b/app/workers/import_issues_csv_worker.rb @@ -5,6 +5,7 @@ class ImportIssuesCsvWorker feature_category :issue_tracking worker_resource_boundary :cpu + weight 2 sidekiq_retries_exhausted do |job| Upload.find(job['args'][2]).destroy diff --git a/app/workers/invalid_gpg_signature_update_worker.rb b/app/workers/invalid_gpg_signature_update_worker.rb index 573efdf9fb1..e1c2eefbf0f 100644 --- a/app/workers/invalid_gpg_signature_update_worker.rb +++ b/app/workers/invalid_gpg_signature_update_worker.rb @@ -4,6 +4,7 @@ class InvalidGpgSignatureUpdateWorker include ApplicationWorker feature_category :source_code_management + weight 2 # rubocop: disable CodeReuse/ActiveRecord def perform(gpg_key_id) diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb index ed88c57e8d4..48bc205113f 100644 --- a/app/workers/merge_worker.rb +++ b/app/workers/merge_worker.rb @@ -5,6 +5,7 @@ class MergeWorker feature_category :source_code_management latency_sensitive_worker! + weight 5 def perform(merge_request_id, current_user_id, params) params = params.with_indifferent_access diff --git a/app/workers/new_issue_worker.rb b/app/workers/new_issue_worker.rb index af9ca332d3c..d696165b447 100644 --- a/app/workers/new_issue_worker.rb +++ b/app/workers/new_issue_worker.rb @@ -7,6 +7,7 @@ class NewIssueWorker feature_category :issue_tracking latency_sensitive_worker! worker_resource_boundary :cpu + weight 2 def perform(issue_id, user_id) return unless objects_found?(issue_id, user_id) diff --git a/app/workers/new_merge_request_worker.rb b/app/workers/new_merge_request_worker.rb index aa3f85c157b..e31ddae1f13 100644 --- a/app/workers/new_merge_request_worker.rb +++ b/app/workers/new_merge_request_worker.rb @@ -7,6 +7,7 @@ class NewMergeRequestWorker feature_category :source_code_management latency_sensitive_worker! worker_resource_boundary :cpu + weight 2 def perform(merge_request_id, user_id) return unless objects_found?(merge_request_id, user_id) diff --git a/app/workers/new_note_worker.rb b/app/workers/new_note_worker.rb index 2a5988a7e32..b446e376007 100644 --- a/app/workers/new_note_worker.rb +++ b/app/workers/new_note_worker.rb @@ -6,6 +6,7 @@ class NewNoteWorker feature_category :issue_tracking latency_sensitive_worker! worker_resource_boundary :cpu + weight 2 # Keep extra parameter to preserve backwards compatibility with # old `NewNoteWorker` jobs (can remove later) diff --git a/app/workers/new_release_worker.rb b/app/workers/new_release_worker.rb index a3a882f9343..edfdb2d7aff 100644 --- a/app/workers/new_release_worker.rb +++ b/app/workers/new_release_worker.rb @@ -5,6 +5,7 @@ class NewReleaseWorker queue_namespace :notifications feature_category :release_orchestration + weight 2 def perform(release_id) release = Release.preloaded.find_by_id(release_id) diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index 334a98a0017..d5038f1152b 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -6,6 +6,7 @@ class PostReceive feature_category :source_code_management latency_sensitive_worker! worker_resource_boundary :cpu + weight 5 def perform(gl_repository, identifier, changes, push_options = {}) project, repo_type = Gitlab::GlRepository.parse(gl_repository) diff --git a/app/workers/process_commit_worker.rb b/app/workers/process_commit_worker.rb index 36af51d859e..ca2896946c9 100644 --- a/app/workers/process_commit_worker.rb +++ b/app/workers/process_commit_worker.rb @@ -12,6 +12,7 @@ class ProcessCommitWorker feature_category :source_code_management latency_sensitive_worker! + weight 3 # project_id - The ID of the project this commit belongs to. # user_id - The ID of the user that pushed the commit. diff --git a/app/workers/rebase_worker.rb b/app/workers/rebase_worker.rb index fd182125c07..ddf5c31a1c2 100644 --- a/app/workers/rebase_worker.rb +++ b/app/workers/rebase_worker.rb @@ -6,6 +6,7 @@ class RebaseWorker include ApplicationWorker feature_category :source_code_management + weight 2 def perform(merge_request_id, current_user_id, skip_ci = false) current_user = User.find(current_user_id) diff --git a/app/workers/remote_mirror_notification_worker.rb b/app/workers/remote_mirror_notification_worker.rb index 8bc19230caf..706131d4e4b 100644 --- a/app/workers/remote_mirror_notification_worker.rb +++ b/app/workers/remote_mirror_notification_worker.rb @@ -4,6 +4,7 @@ class RemoteMirrorNotificationWorker include ApplicationWorker feature_category :source_code_management + weight 2 def perform(remote_mirror_id) remote_mirror = RemoteMirror.find_by_id(remote_mirror_id) diff --git a/app/workers/update_external_pull_requests_worker.rb b/app/workers/update_external_pull_requests_worker.rb index 8b0952528fa..e363b33f1b9 100644 --- a/app/workers/update_external_pull_requests_worker.rb +++ b/app/workers/update_external_pull_requests_worker.rb @@ -4,6 +4,7 @@ class UpdateExternalPullRequestsWorker include ApplicationWorker feature_category :source_code_management + weight 3 def perform(project_id, user_id, ref) project = Project.find_by_id(project_id) diff --git a/app/workers/update_merge_requests_worker.rb b/app/workers/update_merge_requests_worker.rb index acb95353983..ec9739e8a11 100644 --- a/app/workers/update_merge_requests_worker.rb +++ b/app/workers/update_merge_requests_worker.rb @@ -6,6 +6,7 @@ class UpdateMergeRequestsWorker feature_category :source_code_management latency_sensitive_worker! worker_resource_boundary :cpu + weight 3 LOG_TIME_THRESHOLD = 90 # seconds diff --git a/changelogs/unreleased/14707-add-waf-anomaly-summary-service.yml b/changelogs/unreleased/14707-add-waf-anomaly-summary-service.yml new file mode 100644 index 00000000000..18a9bfb67ab --- /dev/null +++ b/changelogs/unreleased/14707-add-waf-anomaly-summary-service.yml @@ -0,0 +1,5 @@ +--- +title: Add WAF Anomaly Summary service +merge_request: 22736 +author: +type: added diff --git a/changelogs/unreleased/194164-add-missing-detailed-sentry-error-graphql.yml b/changelogs/unreleased/194164-add-missing-detailed-sentry-error-graphql.yml new file mode 100644 index 00000000000..163309e32d1 --- /dev/null +++ b/changelogs/unreleased/194164-add-missing-detailed-sentry-error-graphql.yml @@ -0,0 +1,5 @@ +--- +title: Add tags, external_base_url, gitlab_issue to Sentry Detailed Error graphql +merge_request: 23483 +author: +type: added diff --git a/changelogs/unreleased/197500-edit-and-web-ide-look-like-segmented-control-in-file-header.yml b/changelogs/unreleased/197500-edit-and-web-ide-look-like-segmented-control-in-file-header.yml new file mode 100644 index 00000000000..7eb6ea5f521 --- /dev/null +++ b/changelogs/unreleased/197500-edit-and-web-ide-look-like-segmented-control-in-file-header.yml @@ -0,0 +1,5 @@ +--- +title: Remove button group for edit and web ide in file header +merge_request: 23291 +author: +type: other diff --git a/changelogs/unreleased/23296-add-ci-variables-server-port-protocol.yml b/changelogs/unreleased/23296-add-ci-variables-server-port-protocol.yml new file mode 100644 index 00000000000..5ee9fbe540e --- /dev/null +++ b/changelogs/unreleased/23296-add-ci-variables-server-port-protocol.yml @@ -0,0 +1,5 @@ +--- +title: Add CI variables to provide GitLab port and protocol +merge_request: 23296 +author: Aidin Abedi +type: added diff --git a/changelogs/unreleased/broadcast-api-auth.yml b/changelogs/unreleased/broadcast-api-auth.yml new file mode 100644 index 00000000000..57e0097785d --- /dev/null +++ b/changelogs/unreleased/broadcast-api-auth.yml @@ -0,0 +1,5 @@ +--- +title: Allow users to read broadcast messages via API +merge_request: 23298 +author: Rajendra Kadam +type: changed diff --git a/changelogs/unreleased/fix-missing-apply-sugegstion-project-setting.yml b/changelogs/unreleased/fix-missing-apply-sugegstion-project-setting.yml new file mode 100644 index 00000000000..d084148fbd8 --- /dev/null +++ b/changelogs/unreleased/fix-missing-apply-sugegstion-project-setting.yml @@ -0,0 +1,5 @@ +--- +title: Add accidentally deleted project config for custom apply suggestions +merge_request: 23687 +author: Fabio Huser +type: fixed diff --git a/changelogs/unreleased/integrate-gitaly-disk_statistics-grpc.yml b/changelogs/unreleased/integrate-gitaly-disk_statistics-grpc.yml new file mode 100644 index 00000000000..ef90fc89d8d --- /dev/null +++ b/changelogs/unreleased/integrate-gitaly-disk_statistics-grpc.yml @@ -0,0 +1,5 @@ +--- +title: "Support retrieval of disk statistics from Gitaly" +merge_request: 22226 +author: Nels Nelson +type: added diff --git a/config/initializers/lograge.rb b/config/initializers/lograge.rb index 9135073eaea..e7bab1c8fa3 100644 --- a/config/initializers/lograge.rb +++ b/config/initializers/lograge.rb @@ -15,6 +15,10 @@ unless Gitlab::Runtime.sidekiq? data end + # This isn't a user-reachable controller; we use it to check for a + # valid CSRF token in the API + config.lograge.ignore_actions = ['Gitlab::RequestForgeryProtection::Controller#index'] + # Add request parameters to log output config.lograge.custom_options = lambda do |event| params = event.payload[:params] diff --git a/config/routes/git_http.rb b/config/routes/git_http.rb index aac6d418a92..593f818e434 100644 --- a/config/routes/git_http.rb +++ b/config/routes/git_http.rb @@ -1,43 +1,51 @@ -scope(path: '*namespace_id/:project_id', - format: nil, - constraints: { namespace_id: Gitlab::PathRegex.full_namespace_route_regex }) do - scope(constraints: { project_id: Gitlab::PathRegex.project_git_route_regex }, module: :projects) do - # Git HTTP clients ('git clone' etc.) - scope(controller: :git_http) do - get '/info/refs', action: :info_refs - post '/git-upload-pack', action: :git_upload_pack - post '/git-receive-pack', action: :git_receive_pack - end +concern :gitactionable do + scope(controller: :git_http) do + get '/info/refs', action: :info_refs + post '/git-upload-pack', action: :git_upload_pack + post '/git-receive-pack', action: :git_receive_pack + end +end - # Git LFS API (metadata) - scope(path: 'info/lfs/objects', controller: :lfs_api) do - post :batch - post '/', action: :deprecated - get '/*oid', action: :deprecated - end +concern :lfsable do + # Git LFS API (metadata) + scope(path: 'info/lfs/objects', controller: :lfs_api) do + post :batch + post '/', action: :deprecated + get '/*oid', action: :deprecated + end - scope(path: 'info/lfs') do - resources :lfs_locks, controller: :lfs_locks_api, path: 'locks' do - post :unlock, on: :member - post :verify, on: :collection - end + scope(path: 'info/lfs') do + resources :lfs_locks, controller: :lfs_locks_api, path: 'locks' do + post :unlock, on: :member + post :verify, on: :collection end + end - # GitLab LFS object storage - scope(path: 'gitlab-lfs/objects/*oid', controller: :lfs_storage, constraints: { oid: /[a-f0-9]{64}/ }) do - get '/', action: :download + # GitLab LFS object storage + scope(path: 'gitlab-lfs/objects/*oid', controller: :lfs_storage, constraints: { oid: /[a-f0-9]{64}/ }) do + get '/', action: :download - scope constraints: { size: /[0-9]+/ } do - put '/*size/authorize', action: :upload_authorize - put '/*size', action: :upload_finalize - end + scope constraints: { size: /[0-9]+/ } do + put '/*size/authorize', action: :upload_authorize + put '/*size', action: :upload_finalize + end + end +end + +scope(path: '*namespace_id/:repository_id', + format: nil, + constraints: { namespace_id: Gitlab::PathRegex.full_namespace_route_regex }) do + scope(constraints: { repository_id: Gitlab::PathRegex.project_git_route_regex }) do + scope(module: :repositories) do + concerns :gitactionable + concerns :lfsable end end # Redirect /group/project.wiki.git to the project wiki - scope(format: true, constraints: { project_id: Gitlab::PathRegex.project_wiki_git_route_regex, format: :git }) do + scope(format: true, constraints: { repository_id: Gitlab::PathRegex.project_wiki_git_route_regex, format: :git }) do wiki_redirect = redirect do |params, request| - project_id = params[:project_id].delete_suffix('.wiki') + project_id = params[:repository_id].delete_suffix('.wiki') path = [params[:namespace_id], project_id, 'wikis'].join('/') path << "?#{request.query_string}" unless request.query_string.blank? path @@ -47,7 +55,7 @@ scope(path: '*namespace_id/:project_id', end # Redirect /group/project/info/refs to /group/project.git/info/refs - scope(constraints: { project_id: Gitlab::PathRegex.project_route_regex }) do + scope(constraints: { repository_id: Gitlab::PathRegex.project_route_regex }) do # Allow /info/refs, /info/refs?service=git-upload-pack, and # /info/refs?service=git-receive-pack, but nothing else. # @@ -58,7 +66,7 @@ scope(path: '*namespace_id/:project_id', end ref_redirect = redirect do |params, request| - path = "#{params[:namespace_id]}/#{params[:project_id]}.git/info/refs" + path = "#{params[:namespace_id]}/#{params[:repository_id]}.git/info/refs" path << "?#{request.query_string}" unless request.query_string.blank? path end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index e341c91899b..4f8d93e66bb 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -1,7 +1,12 @@ +# This file is generated automatically by +# bin/rake gitlab:sidekiq:sidekiq_queues_yml:generate +# +# Do not edit it manually! +# # This configuration file should be exclusively used to set queue settings for # Sidekiq. Any other setting should be specified using the Sidekiq CLI or the # Sidekiq Ruby API (see config/initializers/sidekiq.rb). ---- +# # All the queues to process and their weights. Every queue _must_ have a weight # defined. # @@ -17,116 +22,217 @@ # to perform) is: # # chance = (queue weight / total weight of all queues) * 100 +--- :queues: - - [post_receive, 5] - - [merge, 5] - - [update_merge_requests, 3] - - [process_commit, 3] - - [new_note, 2] - - [new_issue, 2] - - [notifications, 2] - - [new_merge_request, 2] - - [pipeline_processing, 5] - - [pipeline_creation, 4] - - [pipeline_default, 3] - - [pipeline_cache, 3] - - [deployment, 3] - - [auto_merge, 3] - - [pipeline_hooks, 2] - - [gitlab_shell, 2] - - [email_receiver, 2] - - [emails_on_push, 2] - - [mailers, 2] - - [mail_scheduler, 2] - - [invalid_gpg_signature_update, 2] - - [create_gpg_signature, 2] - - [rebase, 2] - - [upload_checksum, 1] - - [repository_fork, 1] - - [repository_import, 1] - - [github_importer, 1] - - [github_import_advance_stage, 1] - - [project_service, 1] - - [delete_user, 1] - - [todos_destroyer, 1] - - [delete_merged_branches, 1] - - [authorized_projects, 2] - - [expire_build_instance_artifacts, 1] - - [group_destroy, 1] - - [irker, 1] - - [namespaceless_project_destroy, 1] - - [project_cache, 1] - - [project_destroy, 1] - - [project_export, 1] - - [web_hook, 1] - - [repository_check, 1] - - [git_garbage_collect, 1] - - [reactive_caching, 1] - - [cronjob, 1] - - [default, 1] - - [pages, 1] - - [system_hook_push, 1] - - [propagate_service_template, 1] - - [background_migration, 1] - - [gcp_cluster, 1] - - [project_migrate_hashed_storage, 1] - - [project_rollback_hashed_storage, 1] - - [hashed_storage, 1] - - [pages_domain_verification, 1] - - [pages_domain_ssl_renewal, 1] - - [object_storage_upload, 1] - - [object_storage, 1] - - [file_hook, 1] - - [pipeline_background, 1] - - [repository_update_remote_mirror, 1] - - [repository_remove_remote, 1] - - [create_note_diff_file, 1] - - [delete_diff_files, 1] - - [detect_repository_languages, 1] - - [auto_devops, 2] - - [container_repository, 1] - - [object_pool, 1] - - [repository_cleanup, 1] - - [delete_stored_files, 1] - - [remote_mirror_notification, 2] - - [project_daily_statistics, 1] - - [import_issues_csv, 2] - - [chat_notification, 2] - - [migrate_external_diffs, 1] - - [update_project_statistics, 1] - - [phabricator_import_import_tasks, 1] - - [update_namespace_statistics, 1] - - [chaos, 2] - - [create_evidence, 2] - - [group_export, 1] - - [self_monitoring_project_create, 2] - - [self_monitoring_project_delete, 2] - - [error_tracking_issue_link, 2] - - [merge_request_mergeability_check, 5] - - # EE-specific queues - - [analytics, 1] - - [ldap_group_sync, 2] - - [create_github_webhook, 2] - - [geo, 1] - - [repository_update_mirror, 1] - - [repository_push_audit_event, 1] - - [new_epic, 2] - - [project_import_schedule, 1] - - [project_update_repository_storage, 1] - - [admin_emails, 1] - - [elastic_batch_project_indexer, 1] - - [elastic_indexer, 1] - - [elastic_full_index, 1] - - [elastic_commit_indexer, 1] - - [elastic_namespace_indexer, 1] - - [elastic_namespace_rollout, 1] - - [export_csv, 1] - - [incident_management, 2] - - [jira_connect, 1] - - [update_external_pull_requests, 3] - - [refresh_license_compliance_checks, 2] - - [design_management_new_version, 1] - - [epics, 2] - - [personal_access_tokens, 1] - - [adjourned_project_deletion, 1] +- - adjourned_project_deletion + - 1 +- - admin_emails + - 1 +- - authorized_projects + - 2 +- - auto_devops + - 2 +- - auto_merge + - 3 +- - background_migration + - 1 +- - chaos + - 2 +- - chat_notification + - 2 +- - container_repository + - 1 +- - create_evidence + - 2 +- - create_github_webhook + - 2 +- - create_gpg_signature + - 2 +- - create_note_diff_file + - 1 +- - cronjob + - 1 +- - default + - 1 +- - delete_diff_files + - 1 +- - delete_merged_branches + - 1 +- - delete_stored_files + - 1 +- - delete_user + - 1 +- - deployment + - 3 +- - design_management_new_version + - 1 +- - detect_repository_languages + - 1 +- - elastic_batch_project_indexer + - 1 +- - elastic_commit_indexer + - 1 +- - elastic_full_index + - 1 +- - elastic_indexer + - 1 +- - elastic_namespace_indexer + - 1 +- - elastic_namespace_rollout + - 1 +- - email_receiver + - 2 +- - emails_on_push + - 2 +- - epics + - 2 +- - error_tracking_issue_link + - 1 +- - expire_build_instance_artifacts + - 1 +- - export_csv + - 1 +- - file_hook + - 1 +- - gcp_cluster + - 1 +- - geo + - 1 +- - git_garbage_collect + - 1 +- - github_import_advance_stage + - 1 +- - github_importer + - 1 +- - gitlab_shell + - 2 +- - group_destroy + - 1 +- - group_export + - 1 +- - hashed_storage + - 1 +- - import_issues_csv + - 2 +- - incident_management + - 2 +- - invalid_gpg_signature_update + - 2 +- - irker + - 1 +- - jira_connect + - 1 +- - ldap_group_sync + - 2 +- - mail_scheduler + - 2 +- - mailers + - 2 +- - merge + - 5 +- - merge_request_mergeability_check + - 1 +- - migrate_external_diffs + - 1 +- - namespaceless_project_destroy + - 1 +- - new_epic + - 2 +- - new_issue + - 2 +- - new_merge_request + - 2 +- - new_note + - 2 +- - notifications + - 2 +- - object_pool + - 1 +- - object_storage + - 1 +- - pages + - 1 +- - pages_domain_ssl_renewal + - 1 +- - pages_domain_verification + - 1 +- - personal_access_tokens + - 1 +- - phabricator_import_import_tasks + - 1 +- - pipeline_background + - 1 +- - pipeline_cache + - 3 +- - pipeline_creation + - 4 +- - pipeline_default + - 3 +- - pipeline_hooks + - 2 +- - pipeline_processing + - 5 +- - post_receive + - 5 +- - process_commit + - 3 +- - project_cache + - 1 +- - project_daily_statistics + - 1 +- - project_destroy + - 1 +- - project_export + - 1 +- - project_import_schedule + - 1 +- - project_service + - 1 +- - project_update_repository_storage + - 1 +- - propagate_service_template + - 1 +- - reactive_caching + - 1 +- - rebase + - 2 +- - refresh_license_compliance_checks + - 2 +- - remote_mirror_notification + - 2 +- - repository_check + - 1 +- - repository_cleanup + - 1 +- - repository_fork + - 1 +- - repository_import + - 1 +- - repository_push_audit_event + - 1 +- - repository_remove_remote + - 1 +- - repository_update_mirror + - 1 +- - repository_update_remote_mirror + - 1 +- - self_monitoring_project_create + - 2 +- - self_monitoring_project_delete + - 2 +- - system_hook_push + - 1 +- - todos_destroyer + - 1 +- - update_external_pull_requests + - 3 +- - update_merge_requests + - 3 +- - update_namespace_statistics + - 1 +- - update_project_statistics + - 1 +- - upload_checksum + - 1 +- - web_hook + - 1 diff --git a/db/migrate/20200124053531_add_source_to_import_failures.rb b/db/migrate/20200124053531_add_source_to_import_failures.rb new file mode 100644 index 00000000000..532d5803c74 --- /dev/null +++ b/db/migrate/20200124053531_add_source_to_import_failures.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddSourceToImportFailures < ActiveRecord::Migration[5.2] + DOWNTIME = false + + def change + add_column :import_failures, :source, :string, limit: 128 + end +end diff --git a/db/schema.rb b/db/schema.rb index 8ad4e080fe7..8f8a44f1519 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_01_23_155929) do +ActiveRecord::Schema.define(version: 2020_01_24_053531) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -2046,6 +2046,7 @@ ActiveRecord::Schema.define(version: 2020_01_23_155929) do t.string "exception_message", limit: 255 t.integer "retry_count" t.integer "group_id" + t.string "source", limit: 128 t.index ["correlation_id_value"], name: "index_import_failures_on_correlation_id_value" t.index ["group_id"], name: "index_import_failures_on_group_id_not_null", where: "(group_id IS NOT NULL)" t.index ["project_id"], name: "index_import_failures_on_project_id_not_null", where: "(project_id IS NOT NULL)" diff --git a/doc/api/broadcast_messages.md b/doc/api/broadcast_messages.md index ce68eec87ff..6a9a0ba745a 100644 --- a/doc/api/broadcast_messages.md +++ b/doc/api/broadcast_messages.md @@ -4,7 +4,7 @@ Broadcast messages API operates on [broadcast messages](../user/admin_area/broadcast_messages.md). -The broadcast message API is only accessible to administrators. All requests by: +As of GitLab 12.8, GET requests do not require authentication. All other broadcast message API endpoints are accessible only to administrators. Non-GET requests by: - Guests will result in `401 Unauthorized`. - Regular users will result in `403 Forbidden`. @@ -20,7 +20,7 @@ GET /broadcast_messages Example request: ```sh -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/broadcast_messages +curl https://gitlab.example.com/api/v4/broadcast_messages ``` Example response: @@ -57,7 +57,7 @@ Parameters: Example request: ```sh -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/broadcast_messages/1 +curl https://gitlab.example.com/api/v4/broadcast_messages/1 ``` Example response: diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index 4e083142514..39b34e72e24 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -6066,6 +6066,11 @@ type SentryDetailedError { culprit: String! """ + External Base URL of the Sentry Instance + """ + externalBaseUrl: String! + + """ External URL of the error """ externalUrl: String! @@ -6101,6 +6106,11 @@ type SentryDetailedError { gitlabCommitPath: String """ + URL of GitLab Issue + """ + gitlabIssuePath: String + + """ ID (global ID) of the error """ id: ID! @@ -6156,6 +6166,11 @@ type SentryDetailedError { status: SentryErrorStatus! """ + Tags associated with the Sentry Error + """ + tags: SentryErrorTags! + + """ Title of the error """ title: String! @@ -6209,6 +6224,21 @@ enum SentryErrorStatus { } """ +State of a Sentry error +""" +type SentryErrorTags { + """ + Severity level of the Sentry Error + """ + level: String + + """ + Logger of the Sentry Error + """ + logger: String +} + +""" Represents a snippet entry """ type Snippet implements Noteable { diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index 5c1c05d6d4e..2d2bcaf32bd 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -16747,6 +16747,24 @@ "deprecationReason": null }, { + "name": "externalBaseUrl", + "description": "External Base URL of the Sentry Instance", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "externalUrl", "description": "External URL of the error", "args": [ @@ -16865,6 +16883,20 @@ "deprecationReason": null }, { + "name": "gitlabIssuePath", + "description": "URL of GitLab Issue", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "id", "description": "ID (global ID) of the error", "args": [ @@ -17051,6 +17083,24 @@ "deprecationReason": null }, { + "name": "tags", + "description": "Tags associated with the Sentry Error", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "SentryErrorTags", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "title", "description": "Title of the error", "args": [ @@ -17198,6 +17248,47 @@ }, { "kind": "OBJECT", + "name": "SentryErrorTags", + "description": "State of a Sentry error", + "fields": [ + { + "name": "level", + "description": "Severity level of the Sentry Error", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "logger", + "description": "Logger of the Sentry Error", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "GrafanaIntegration", "description": null, "fields": [ diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 210554243a8..790e55d437f 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -923,6 +923,7 @@ Autogenerated return type of RemoveAwardEmoji | --- | ---- | ---------- | | `count` | Int! | Count of occurrences | | `culprit` | String! | Culprit of the error | +| `externalBaseUrl` | String! | External Base URL of the Sentry Instance | | `externalUrl` | String! | External URL of the error | | `firstReleaseLastCommit` | String | Commit the error was first seen | | `firstReleaseShortVersion` | String | Release version the error was first seen | @@ -930,6 +931,7 @@ Autogenerated return type of RemoveAwardEmoji | `frequency` | SentryErrorFrequency! => Array | Last 24hr stats of the error | | `gitlabCommit` | String | GitLab commit SHA attributed to the Error based on the release version | | `gitlabCommitPath` | String | Path to the GitLab page for the GitLab commit attributed to the error | +| `gitlabIssuePath` | String | URL of GitLab Issue | | `id` | ID! | ID (global ID) of the error | | `lastReleaseLastCommit` | String | Commit the error was last seen | | `lastReleaseShortVersion` | String | Release version the error was last seen | @@ -941,6 +943,7 @@ Autogenerated return type of RemoveAwardEmoji | `sentryProjectSlug` | String! | Slug of the project affected by the error | | `shortId` | String! | Short ID (Sentry ID) of the error | | `status` | SentryErrorStatus! | Status of the error | +| `tags` | SentryErrorTags! | Tags associated with the Sentry Error | | `title` | String! | Title of the error | | `type` | String! | Type of the error | | `userCount` | Int! | Count of users affected by the error | @@ -952,6 +955,15 @@ Autogenerated return type of RemoveAwardEmoji | `count` | Int! | Count of errors received since the previously recorded time | | `time` | Time! | Time the error frequency stats were recorded | +## SentryErrorTags + +State of a Sentry error + +| Name | Type | Description | +| --- | ---- | ---------- | +| `level` | String | Severity level of the Sentry Error | +| `logger` | String | Logger of the Sentry Error | + ## Snippet Represents a snippet entry diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 19fcca36604..8f1ce48906a 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -292,6 +292,8 @@ export CI_RUNNER_TAGS="docker, linux" export CI_SERVER="yes" export CI_SERVER_URL="https://example.com" export CI_SERVER_HOST="example.com" +export CI_SERVER_PORT="443" +export CI_SERVER_PROTOCOL="https" export CI_SERVER_NAME="GitLab" export CI_SERVER_REVISION="70606bf" export CI_SERVER_VERSION="8.9.0" @@ -686,6 +688,10 @@ if [[ -d "/builds/gitlab-examples/ci-debug-trace/.git" ]]; then ++ CI_SERVER_URL=https://gitlab.com:3000 ++ export CI_SERVER_HOST=gitlab.com ++ CI_SERVER_HOST=gitlab.com +++ export CI_SERVER_PORT=3000 +++ CI_SERVER_PORT=3000 +++ export CI_SERVER_PROTOCOL=https +++ CI_SERVER_PROTOCOL=https ++ export CI_SERVER_NAME=GitLab ++ CI_SERVER_NAME=GitLab ++ export CI_SERVER_VERSION=12.6.0-pre diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index 24a16fe6a70..5cc93427d42 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -114,6 +114,8 @@ future GitLab releases.** | `CI_SERVER` | all | all | Mark that job is executed in CI environment | | `CI_SERVER_URL` | 12.7 | all | The base URL of the GitLab instance, including protocol and port (like `https://gitlab.example.com:8080`) | | `CI_SERVER_HOST` | 12.1 | all | Host component of the GitLab instance URL, without protocol and port (like `gitlab.example.com`) | +| `CI_SERVER_PORT` | 12.8 | all | Port component of the GitLab instance URL, without host and protocol (like `3000`) | +| `CI_SERVER_PROTOCOL` | 12.8 | all | Protocol component of the GitLab instance URL, without host and port (like `https`) | | `CI_SERVER_NAME` | all | all | The name of CI server that is used to coordinate jobs | | `CI_SERVER_REVISION` | all | all | GitLab revision that is used to schedule jobs | | `CI_SERVER_VERSION` | all | all | GitLab version that is used to schedule jobs | diff --git a/doc/user/discussions/img/suggestion_code_block_editor_v12_8.png b/doc/user/discussions/img/suggestion_code_block_editor_v12_8.png Binary files differnew file mode 100644 index 00000000000..927b4f812a5 --- /dev/null +++ b/doc/user/discussions/img/suggestion_code_block_editor_v12_8.png diff --git a/doc/user/discussions/img/suggestion_code_block_output_v12_8.png b/doc/user/discussions/img/suggestion_code_block_output_v12_8.png Binary files differnew file mode 100644 index 00000000000..74833253aa0 --- /dev/null +++ b/doc/user/discussions/img/suggestion_code_block_output_v12_8.png diff --git a/doc/user/discussions/index.md b/doc/user/discussions/index.md index 6016837a769..1855c6666e6 100644 --- a/doc/user/discussions/index.md +++ b/doc/user/discussions/index.md @@ -439,6 +439,16 @@ Suggestions covering multiple lines are limited to 100 lines _above_ and 100 lines _below_ the commented diff line, allowing up to 200 changed lines per suggestion. +### Code block nested in Suggestions + +If you need to make a suggestion that involves a +[fenced code block](../markdown.md#code-spans-and-blocks), wrap your suggestion in four backticks +instead of the usual three. + + + + + ### Configure the commit message for applied Suggestions > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13086) in GitLab 12.7. diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index 994e12445b7..48d4f1a0a63 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -4,9 +4,6 @@ module API class BroadcastMessages < Grape::API include PaginationParams - before { authenticate! } - before { authenticated_as_admin! } - resource :broadcast_messages do helpers do def find_message @@ -40,6 +37,8 @@ module API optional :target_path, type: String, desc: 'Target path' end post do + authenticated_as_admin! + message = BroadcastMessage.create(declared_params(include_missing: false)) if message.persisted? @@ -76,6 +75,8 @@ module API optional :target_path, type: String, desc: 'Target path' end put ':id' do + authenticated_as_admin! + message = find_message if message.update(declared_params(include_missing: false)) @@ -93,6 +94,8 @@ module API requires :id, type: Integer, desc: 'Broadcast message ID' end delete ':id' do + authenticated_as_admin! + message = find_message destroy_conditionally!(message) diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index c8e96c7c5db..cc9d45dcae4 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -104,14 +104,12 @@ module API # rubocop:disable Gitlab/ModuleWithInstanceVariables def set_project - if params[:gl_repository] - @project, @repo_type = Gitlab::GlRepository.parse(params[:gl_repository]) - @redirected_path = nil - elsif params[:project] - @project, @repo_type, @redirected_path = Gitlab::RepoPath.parse(params[:project]) - else - @project, @repo_type, @redirected_path = nil, nil, nil - end + @project, @repo_type, @redirected_path = + if params[:gl_repository] + Gitlab::GlRepository.parse(params[:gl_repository]) + elsif params[:project] + Gitlab::RepoPath.parse(params[:project]) + end end # rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index d41490d2ebd..3e9cf2ab320 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -4,7 +4,7 @@ module Constraints class ProjectUrlConstrainer def matches?(request, existence_check: true) namespace_path = request.params[:namespace_id] - project_path = request.params[:project_id] || request.params[:id] + project_path = request.params[:project_id] || request.params[:id] || request.params[:repository_id] full_path = [namespace_path, project_path].join('/') return false unless ProjectPathValidator.valid_path?(full_path) diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index c225ca56c55..96062ae87bc 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -9,7 +9,6 @@ class Feature %w[ cache_invalidator inforef_uploadpack_cache - get_tag_messages_go filter_shas_with_signatures_go commit_without_batch_check ].freeze diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb index 64ab5db4fcd..89a836e629f 100644 --- a/lib/gitaly/server.rb +++ b/lib/gitaly/server.rb @@ -53,6 +53,20 @@ module Gitaly storage_status&.fs_type end + def disk_used + disk_statistics_storage_status&.used + end + + def disk_available + disk_statistics_storage_status&.available + end + + # Simple convenience method for when obtaining both used and available + # statistics at once is preferred. + def disk_stats + disk_statistics_storage_status + end + def address Gitlab::GitalyClient.address(@storage) rescue RuntimeError => e @@ -65,6 +79,10 @@ module Gitaly @storage_status ||= info.storage_statuses.find { |s| s.storage_name == storage } end + def disk_statistics_storage_status + @disk_statistics_storage_status ||= disk_statistics.storage_statuses.find { |s| s.storage_name == storage } + end + def matches_sha? match = server_version.match(SHA_VERSION_REGEX) return false unless match @@ -76,7 +94,19 @@ module Gitaly @info ||= begin Gitlab::GitalyClient::ServerService.new(@storage).info - rescue GRPC::Unavailable, GRPC::DeadlineExceeded + rescue GRPC::Unavailable, GRPC::DeadlineExceeded => ex + Gitlab::ErrorTracking.track_exception(ex) + # This will show the server as being out of date + Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: []) + end + end + + def disk_statistics + @disk_statistics ||= + begin + Gitlab::GitalyClient::ServerService.new(@storage).disk_statistics + rescue GRPC::Unavailable, GRPC::DeadlineExceeded => ex + Gitlab::ErrorTracking.track_exception(ex) # This will show the server as being out of date Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: []) end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 262a1ef653f..4eb1ccf32ba 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -432,10 +432,7 @@ module Gitlab end def self.filesystem_id(storage) - response = Gitlab::GitalyClient::ServerService.new(storage).info - storage_status = response.storage_statuses.find { |status| status.storage_name == storage } - - storage_status&.filesystem_id + Gitlab::GitalyClient::ServerService.new(storage).storage_info&.filesystem_id end def self.filesystem_id_from_disk(storage) @@ -446,6 +443,14 @@ module Gitlab nil end + def self.filesystem_disk_available(storage) + Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.available + end + + def self.filesystem_disk_used(storage) + Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.used + end + def self.timeout(timeout_name) Gitlab::CurrentSettings.current_application_settings[timeout_name] end diff --git a/lib/gitlab/gitaly_client/server_service.rb b/lib/gitlab/gitaly_client/server_service.rb index 0ade6942db9..36bda67c26e 100644 --- a/lib/gitlab/gitaly_client/server_service.rb +++ b/lib/gitlab/gitaly_client/server_service.rb @@ -13,6 +13,24 @@ module Gitlab def info GitalyClient.call(@storage, :server_service, :server_info, Gitaly::ServerInfoRequest.new, timeout: GitalyClient.fast_timeout) end + + def disk_statistics + GitalyClient.call(@storage, :server_service, :disk_statistics, Gitaly::DiskStatisticsRequest.new, timeout: GitalyClient.fast_timeout) + end + + def storage_info + storage_specific(info) + end + + def storage_disk_statistics + storage_specific(disk_statistics) + end + + private + + def storage_specific(response) + response.storage_statuses.find { |status| status.storage_name == @storage } + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2e27e954e79..3db6c3b51c0 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -43,6 +43,9 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend push_frontend_feature_flag(:snippets_vue, default_enabled: false) + push_frontend_feature_flag(:monaco_snippets, default_enabled: false) + push_frontend_feature_flag(:monaco_blobs, default_enabled: false) + push_frontend_feature_flag(:monaco_ci, default_enabled: false) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/import_export/import_failure_service.rb b/lib/gitlab/import_export/import_failure_service.rb index eeaf10870c8..d4eca551b49 100644 --- a/lib/gitlab/import_export/import_failure_service.rb +++ b/lib/gitlab/import_export/import_failure_service.rb @@ -12,9 +12,14 @@ module Gitlab @association = importable.association(:import_failures) end - def with_retry(relation_key, relation_index) + def with_retry(action:, relation_key: nil, relation_index: nil) on_retry = -> (exception, retry_count, *_args) do - log_import_failure(relation_key, relation_index, exception, retry_count) + log_import_failure( + source: action, + relation_key: relation_key, + relation_index: relation_index, + exception: exception, + retry_count: retry_count) end Retriable.with_context(:relation_import, on_retry: on_retry) do @@ -22,8 +27,9 @@ module Gitlab end end - def log_import_failure(relation_key, relation_index, exception, retry_count = 0) + def log_import_failure(source:, relation_key: nil, relation_index: nil, exception:, retry_count: 0) extra = { + source: source, relation_key: relation_key, relation_index: relation_index, retry_count: retry_count diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index e598cfc143e..c4ac6a3a3f2 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -21,7 +21,9 @@ module Gitlab RelationRenameService.rename(@tree_hash) if relation_tree_restorer.restore - @project.merge_requests.set_latest_merge_request_diff_ids! + import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do + @project.merge_requests.set_latest_merge_request_diff_ids! + end true else @@ -72,6 +74,10 @@ module Gitlab def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end + + def import_failure_service + @import_failure_service ||= ImportFailureService.new(@project) + end end end end diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 44cf90fb86a..3606e1c5bd9 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -73,13 +73,17 @@ module Gitlab relation_object.assign_attributes(importable_class_sym => @importable) - import_failure_service.with_retry(relation_key, relation_index) do + import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do relation_object.save! end save_id_mapping(relation_key, data_hash, relation_object) rescue => e - import_failure_service.log_import_failure(relation_key, relation_index, e) + import_failure_service.log_import_failure( + source: 'process_relation_item!', + relation_key: relation_key, + relation_index: relation_index, + exception: e) end def import_failure_service diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index c749816cf6a..2cf2a81f812 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -12,12 +12,12 @@ module Gitlab ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance' WHITELISTED_GIT_ROUTES = { - 'projects/git_http' => %w{git_upload_pack git_receive_pack} + 'repositories/git_http' => %w{git_upload_pack git_receive_pack} }.freeze WHITELISTED_GIT_LFS_ROUTES = { - 'projects/lfs_api' => %w{batch}, - 'projects/lfs_locks_api' => %w{verify create unlock} + 'repositories/lfs_api' => %w{batch}, + 'repositories/lfs_locks_api' => %w{verify create unlock} }.freeze WHITELISTED_GIT_REVISION_ROUTES = { diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index c96212f27a7..c8719023d40 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -6,6 +6,7 @@ module Gitlab module SidekiqConfig FOSS_QUEUE_CONFIG_PATH = 'app/workers/all_queues.yml' EE_QUEUE_CONFIG_PATH = 'ee/app/workers/all_queues.yml' + SIDEKIQ_QUEUES_PATH = 'config/sidekiq_queues.yml' QUEUE_CONFIG_PATHS = [ FOSS_QUEUE_CONFIG_PATH, @@ -13,11 +14,19 @@ module Gitlab ].compact.freeze # For queues that don't have explicit workers - default and mailers - DummyWorker = Struct.new(:queue) + DummyWorker = Struct.new(:queue, :weight) do + def queue_namespace + nil + end + + def get_weight + weight + end + end DEFAULT_WORKERS = [ - Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('default'), ee: false), - Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('mailers'), ee: false) + Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('default', 1), ee: false), + Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('mailers', 2), ee: false) ].freeze class << self @@ -30,7 +39,7 @@ module Gitlab def config_queues @config_queues ||= begin - config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml')) + config = YAML.load_file(Rails.root.join(SIDEKIQ_QUEUES_PATH)) config[:queues].map(&:first) end end @@ -65,6 +74,28 @@ module Gitlab Gitlab.ee? && ee_workers != YAML.safe_load(File.read(EE_QUEUE_CONFIG_PATH)) end + def queues_for_sidekiq_queues_yml + namespaces_with_equal_weights = + workers + .group_by(&:queue_namespace) + .map(&:last) + .select { |workers| workers.map(&:get_weight).uniq.count == 1 } + .map(&:first) + + namespaces = namespaces_with_equal_weights.map(&:queue_namespace).to_set + remaining_queues = workers.reject { |worker| namespaces.include?(worker.queue_namespace) } + + (namespaces_with_equal_weights.map(&:namespace_and_weight) + + remaining_queues.map(&:queue_and_weight)).sort + end + + def sidekiq_queues_yml_outdated? + # YAML.load is OK here as we control the file contents + config_queues = YAML.load(File.read(SIDEKIQ_QUEUES_PATH))[:queues] # rubocop:disable Security/YAMLLoad + + queues_for_sidekiq_queues_yml != config_queues + end + private def find_workers(root, ee:) diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb index 313d9b17b16..ac94bab9a8f 100644 --- a/lib/gitlab/sidekiq_config/worker.rb +++ b/lib/gitlab/sidekiq_config/worker.rb @@ -7,8 +7,9 @@ module Gitlab attr_reader :klass delegate :feature_category_not_owned?, :get_feature_category, - :get_worker_resource_boundary, :latency_sensitive_worker?, - :queue, :worker_has_external_dependencies?, + :get_weight, :get_worker_resource_boundary, + :latency_sensitive_worker?, :queue, :queue_namespace, + :worker_has_external_dependencies?, to: :klass def initialize(klass, ee:) @@ -35,7 +36,7 @@ module Gitlab # Put namespaced queues first def to_sort - [queue.include?(':') ? 0 : 1, queue] + [queue_namespace ? 0 : 1, queue] end # YAML representation @@ -46,6 +47,14 @@ module Gitlab def to_yaml queue end + + def namespace_and_weight + [queue_namespace, get_weight] + end + + def queue_and_weight + [queue, get_weight] + end end end end diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index 15c6c930386..eb3de195626 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -4,8 +4,13 @@ return if Rails.env.production? namespace :gitlab do namespace :sidekiq do + def write_yaml(path, banner, object) + File.write(path, banner + YAML.dump(object)) + end + namespace :all_queues_yml do - def write_yaml(path, object) + desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions' + task generate: :environment do banner = <<~BANNER # This file is generated automatically by # bin/rake gitlab:sidekiq:all_queues_yml:generate @@ -13,17 +18,12 @@ namespace :gitlab do # Do not edit it manually! BANNER - File.write(path, banner + YAML.dump(object)) - end - - desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions' - task generate: :environment do foss_workers, ee_workers = Gitlab::SidekiqConfig.workers_for_all_queues_yml - write_yaml(Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH, foss_workers) + write_yaml(Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH, banner, foss_workers) if Gitlab.ee? - write_yaml(Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH, ee_workers) + write_yaml(Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH, banner, ee_workers) end end @@ -44,5 +44,57 @@ namespace :gitlab do end end end + + namespace :sidekiq_queues_yml do + desc 'GitLab | Sidekiq | Generate sidekiq_queues.yml based on worker definitions' + task generate: :environment do + banner = <<~BANNER + # This file is generated automatically by + # bin/rake gitlab:sidekiq:sidekiq_queues_yml:generate + # + # Do not edit it manually! + # + # This configuration file should be exclusively used to set queue settings for + # Sidekiq. Any other setting should be specified using the Sidekiq CLI or the + # Sidekiq Ruby API (see config/initializers/sidekiq.rb). + # + # All the queues to process and their weights. Every queue _must_ have a weight + # defined. + # + # The available weights are as follows + # + # 1: low priority + # 2: medium priority + # 3: high priority + # 5: _super_ high priority, this should only be used for _very_ important queues + # + # As per http://stackoverflow.com/a/21241357/290102 the formula for calculating + # the likelihood of a job being popped off a queue (given all queues have work + # to perform) is: + # + # chance = (queue weight / total weight of all queues) * 100 + BANNER + + queues_and_weights = Gitlab::SidekiqConfig.queues_for_sidekiq_queues_yml + + write_yaml(Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH, banner, queues: queues_and_weights) + end + + desc 'GitLab | Sidekiq | Validate that sidekiq_queues.yml matches worker definitions' + task check: :environment do + if Gitlab::SidekiqConfig.sidekiq_queues_yml_outdated? + raise <<~MSG + Changes in worker queues found, please update the metadata by running: + + bin/rake gitlab:sidekiq:sidekiq_queues_yml:generate + + Then commit and push the changes from: + + - #{Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH} + + MSG + end + end + end end end diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 91e7031744a..7a4d09bb6d4 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -38,11 +38,13 @@ unless Rails.env.production? ] if Gitlab.ee? - # This task will fail on CE installations (e.g. gitlab-org/gitlab-foss) - # since it will detect strings in the locale files that do not exist in - # the source files. To work around this we will only enable this task on - # EE installations. + # These tasks will fail on FOSS installations + # (e.g. gitlab-org/gitlab-foss) since they test against a single + # file that is generated by an EE installation, which can + # contain values that a FOSS installation won't find. To work + # around this we will only enable this task on EE installations. tasks << 'gettext:updated_check' + tasks << 'gitlab:sidekiq:sidekiq_queues_yml:check' end tasks.each do |task| diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 025d21e1c25..74d7681def3 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -10419,6 +10419,9 @@ msgstr "" msgid "Is using license seat:" msgstr "" +msgid "Is using seat" +msgstr "" + msgid "IssuableStatus|Closed" msgstr "" diff --git a/qa/qa/page/project/job/show.rb b/qa/qa/page/project/job/show.rb index 07dea3449f1..d673efd1970 100644 --- a/qa/qa/page/project/job/show.rb +++ b/qa/qa/page/project/job/show.rb @@ -13,6 +13,10 @@ module QA::Page element :pipeline_path end + view 'app/assets/javascripts/jobs/components/sidebar.vue' do + element :retry_button + end + def successful?(timeout: 60) raise "Timed out waiting for the build trace to load" unless loaded? raise "Timed out waiting for the status to be a valid completed state" unless completed?(timeout: timeout) @@ -33,6 +37,10 @@ module QA::Page result end + def retry! + click_element :retry_button + end + private def loaded?(wait: 60) diff --git a/qa/qa/page/project/settings/ci_variables.rb b/qa/qa/page/project/settings/ci_variables.rb index 64a182e5b3a..2bb285d6086 100644 --- a/qa/qa/page/project/settings/ci_variables.rb +++ b/qa/qa/page/project/settings/ci_variables.rb @@ -52,6 +52,14 @@ module QA end end + def remove_variable(location: :first) + within('.ci-variable-row-body', match: location) do + find('button.ci-variable-row-remove-button').click + end + + save_variables + end + private def toggle_masked(masked_node, masked) diff --git a/qa/qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb b/qa/qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb index 0ca49bd080b..19cd7d123c9 100644 --- a/qa/qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb +++ b/qa/qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb @@ -46,12 +46,7 @@ module QA deploy_key_name = "DEPLOY_KEY_#{key.name}_#{key.bits}" - Resource::CiVariable.fabricate_via_browser_ui! do |resource| - resource.project = @project - resource.key = deploy_key_name - resource.value = key.private_key - resource.masked = false - end + make_ci_variable(deploy_key_name, key) gitlab_ci = <<~YAML cat-config: @@ -90,6 +85,17 @@ module QA expect(job.output).to include(sha1sum) end end + + private + + def make_ci_variable(key_name, key) + Resource::CiVariable.fabricate_via_api! do |resource| + resource.project = @project + resource.key = key_name + resource.value = key.private_key + resource.masked = false + end + end end end end diff --git a/spec/controllers/concerns/lfs_request_spec.rb b/spec/controllers/concerns/lfs_request_spec.rb index 584448e68f9..79257e9a7f6 100644 --- a/spec/controllers/concerns/lfs_request_spec.rb +++ b/spec/controllers/concerns/lfs_request_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' describe LfsRequest do include ProjectForksHelper - controller(Projects::GitHttpClientController) do + controller(Repositories::GitHttpClientController) do # `described_class` is not available in this context include LfsRequest diff --git a/spec/controllers/projects/git_http_controller_spec.rb b/spec/controllers/projects/git_http_controller_spec.rb deleted file mode 100644 index 4df53121aaa..00000000000 --- a/spec/controllers/projects/git_http_controller_spec.rb +++ /dev/null @@ -1,107 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Projects::GitHttpController do - include GitHttpHelpers - - let_it_be(:project) { create(:project, :public, :repository) } - let(:project_params) do - { - namespace_id: project.namespace.to_param, - project_id: project.path + '.git' - } - end - let(:params) { project_params } - - describe 'HEAD #info_refs' do - it 'returns 403' do - head :info_refs, params: { namespace_id: project.namespace.to_param, project_id: project.path + '.git' } - - expect(response.status).to eq(403) - end - end - - describe 'GET #info_refs' do - let(:params) { project_params.merge(service: 'git-upload-pack') } - - it 'returns 401 for unauthenticated requests to public repositories when http protocol is disabled' do - stub_application_setting(enabled_git_access_protocol: 'ssh') - - get :info_refs, params: params - - expect(response.status).to eq(401) - end - - context 'with authorized user' do - let(:user) { project.owner } - - before do - request.headers.merge! auth_env(user.username, user.password, nil) - end - - it 'returns 200' do - get :info_refs, params: params - - expect(response.status).to eq(200) - end - - it 'updates the user activity' do - expect_next_instance_of(Users::ActivityService) do |activity_service| - expect(activity_service).to receive(:execute) - end - - get :info_refs, params: params - end - end - - context 'with exceptions' do - before do - allow(controller).to receive(:verify_workhorse_api!).and_return(true) - end - - it 'returns 503 with GRPC Unavailable' do - allow(controller).to receive(:access_check).and_raise(GRPC::Unavailable) - - get :info_refs, params: params - - expect(response.status).to eq(503) - end - - it 'returns 503 with timeout error' do - allow(controller).to receive(:access_check).and_raise(Gitlab::GitAccess::TimeoutError) - - get :info_refs, params: params - - expect(response.status).to eq(503) - expect(response.body).to eq 'Gitlab::GitAccess::TimeoutError' - end - end - end - - describe 'POST #git_upload_pack' do - before do - allow(controller).to receive(:authenticate_user).and_return(true) - allow(controller).to receive(:verify_workhorse_api!).and_return(true) - allow(controller).to receive(:access_check).and_return(nil) - end - - after do - post :git_upload_pack, params: params - end - - context 'on a read-only instance' do - before do - allow(Gitlab::Database).to receive(:read_only?).and_return(true) - end - - it 'does not update project statistics' do - expect(ProjectDailyStatisticsWorker).not_to receive(:perform_async) - end - end - - it 'updates project statistics' do - expect(ProjectDailyStatisticsWorker).to receive(:perform_async) - end - end -end diff --git a/spec/controllers/repositories/git_http_controller_spec.rb b/spec/controllers/repositories/git_http_controller_spec.rb new file mode 100644 index 00000000000..10a7b72ca89 --- /dev/null +++ b/spec/controllers/repositories/git_http_controller_spec.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Repositories::GitHttpController do + include GitHttpHelpers + + let_it_be(:project) { create(:project, :public, :repository) } + + let(:namespace_id) { project.namespace.to_param } + let(:repository_id) { project.path + '.git' } + let(:project_params) do + { + namespace_id: namespace_id, + repository_id: repository_id + } + end + let(:params) { project_params } + + describe 'HEAD #info_refs' do + it 'returns 403' do + head :info_refs, params: params + + expect(response.status).to eq(403) + end + end + + shared_examples 'info_refs behavior' do + describe 'GET #info_refs' do + let(:params) { project_params.merge(service: 'git-upload-pack') } + + it 'returns 401 for unauthenticated requests to public repositories when http protocol is disabled' do + stub_application_setting(enabled_git_access_protocol: 'ssh') + allow(controller).to receive(:basic_auth_provided?).and_call_original + + expect(controller).to receive(:http_download_allowed?).and_call_original + + get :info_refs, params: params + + expect(response.status).to eq(401) + end + + context 'with authorized user' do + let(:user) { project.owner } + + before do + request.headers.merge! auth_env(user.username, user.password, nil) + end + + it 'returns 200' do + get :info_refs, params: params + + expect(response.status).to eq(200) + end + + it 'updates the user activity' do + expect_next_instance_of(Users::ActivityService) do |activity_service| + expect(activity_service).to receive(:execute) + end + + get :info_refs, params: params + end + end + + context 'with exceptions' do + before do + allow(controller).to receive(:verify_workhorse_api!).and_return(true) + end + + it 'returns 503 with GRPC Unavailable' do + allow(controller).to receive(:access_check).and_raise(GRPC::Unavailable) + + get :info_refs, params: params + + expect(response.status).to eq(503) + end + + it 'returns 503 with timeout error' do + allow(controller).to receive(:access_check).and_raise(Gitlab::GitAccess::TimeoutError) + + get :info_refs, params: params + + expect(response.status).to eq(503) + expect(response.body).to eq 'Gitlab::GitAccess::TimeoutError' + end + end + end + end + + shared_examples 'git_upload_pack behavior' do |expected| + describe 'POST #git_upload_pack' do + before do + allow(controller).to receive(:authenticate_user).and_return(true) + allow(controller).to receive(:verify_workhorse_api!).and_return(true) + allow(controller).to receive(:access_check).and_return(nil) + end + + after do + post :git_upload_pack, params: params + end + + context 'on a read-only instance' do + before do + allow(Gitlab::Database).to receive(:read_only?).and_return(true) + end + + it 'does not update project statistics' do + expect(ProjectDailyStatisticsWorker).not_to receive(:perform_async) + end + end + + if expected + it 'updates project statistics' do + expect(ProjectDailyStatisticsWorker).to receive(:perform_async) + end + else + it 'does not update project statistics' do + expect(ProjectDailyStatisticsWorker).not_to receive(:perform_async) + end + end + end + end + + shared_examples 'access checker class' do + let(:params) { project_params.merge(service: 'git-upload-pack') } + + it 'calls the right access class checker with the right object' do + allow(controller).to receive(:verify_workhorse_api!).and_return(true) + + access_double = double + expect(expected_class).to receive(:new).with(anything, expected_object, 'http', anything).and_return(access_double) + allow(access_double).to receive(:check).and_return(false) + + get :info_refs, params: params + end + end + + context 'when repository container is a project' do + it_behaves_like 'info_refs behavior' + it_behaves_like 'git_upload_pack behavior', true + it_behaves_like 'access checker class' do + let(:expected_class) { Gitlab::GitAccess } + let(:expected_object) { project } + end + end +end diff --git a/spec/graphql/types/error_tracking/sentry_detailed_error_type_spec.rb b/spec/graphql/types/error_tracking/sentry_detailed_error_type_spec.rb index 30cede6f4cf..3a512fee3b3 100644 --- a/spec/graphql/types/error_tracking/sentry_detailed_error_type_spec.rb +++ b/spec/graphql/types/error_tracking/sentry_detailed_error_type_spec.rb @@ -20,6 +20,7 @@ describe GitlabSchema.types['SentryDetailedError'] do message culprit externalUrl + externalBaseUrl sentryProjectId sentryProjectName sentryProjectSlug @@ -30,8 +31,10 @@ describe GitlabSchema.types['SentryDetailedError'] do lastReleaseLastCommit firstReleaseShortVersion lastReleaseShortVersion + gitlabIssuePath gitlabCommit gitlabCommitPath + tags ] is_expected.to have_graphql_fields(*expected_fields) diff --git a/spec/lib/gitaly/server_spec.rb b/spec/lib/gitaly/server_spec.rb index 184d049d1fb..5142f705251 100644 --- a/spec/lib/gitaly/server_spec.rb +++ b/spec/lib/gitaly/server_spec.rb @@ -66,6 +66,53 @@ describe Gitaly::Server do end end + context "when examining disk statistics for a given server" do + let(:disk_available) { 42 } + let(:disk_used) { 42 } + let(:storage_status) { double('storage_status') } + + before do + allow(storage_status).to receive(:storage_name).and_return('default') + allow(storage_status).to receive(:available).and_return(disk_available) + allow(storage_status).to receive(:used).and_return(disk_used) + response = double("response") + allow(response).to receive(:storage_statuses).and_return([storage_status]) + allow_next_instance_of(Gitlab::GitalyClient::ServerService) do |instance| + allow(instance).to receive(:disk_statistics).and_return(response) + end + end + + describe '#disk_available' do + subject { server.disk_available } + + it { is_expected.to be_present } + + it "returns disk available for the storage of the instantiated server" do + is_expected.to eq(disk_available) + end + end + + describe '#disk_used' do + subject { server.disk_used } + + it { is_expected.to be_present } + + it "returns disk used for the storage of the instantiated server" do + is_expected.to eq(disk_used) + end + end + + describe '#disk_stats' do + subject { server.disk_stats } + + it { is_expected.to be_present } + + it "returns the storage of the instantiated server" do + is_expected.to eq(storage_status) + end + end + end + describe '#expected_version?' do using RSpec::Parameterized::TableSyntax diff --git a/spec/lib/gitlab/gitaly_client_spec.rb b/spec/lib/gitlab/gitaly_client_spec.rb index ebf56c0ae66..b03c1feb429 100644 --- a/spec/lib/gitlab/gitaly_client_spec.rb +++ b/spec/lib/gitlab/gitaly_client_spec.rb @@ -52,7 +52,7 @@ describe Gitlab::GitalyClient do end describe '.filesystem_id' do - it 'returns an empty string when the storage is not found in the response' do + it 'returns an empty string when the relevant storage status is not found in the response' do response = double("response") allow(response).to receive(:storage_statuses).and_return([]) allow_next_instance_of(Gitlab::GitalyClient::ServerService) do |instance| @@ -63,6 +63,63 @@ describe Gitlab::GitalyClient do end end + context 'when the relevant storage status is not found' do + before do + response = double('response') + allow(response).to receive(:storage_statuses).and_return([]) + allow_next_instance_of(Gitlab::GitalyClient::ServerService) do |instance| + allow(instance).to receive(:disk_statistics).and_return(response) + expect(instance).to receive(:storage_disk_statistics) + end + end + + describe '.filesystem_disk_available' do + it 'returns nil when the relevant storage status is not found in the response' do + expect(described_class.filesystem_disk_available('default')).to eq(nil) + end + end + + describe '.filesystem_disk_used' do + it 'returns nil when the relevant storage status is not found in the response' do + expect(described_class.filesystem_disk_used('default')).to eq(nil) + end + end + end + + context 'when the relevant storage status is found' do + let(:disk_available) { 42 } + let(:disk_used) { 42 } + let(:storage_status) { double('storage_status') } + + before do + allow(storage_status).to receive(:storage_name).and_return('default') + allow(storage_status).to receive(:used).and_return(disk_used) + allow(storage_status).to receive(:available).and_return(disk_available) + response = double('response') + allow(response).to receive(:storage_statuses).and_return([storage_status]) + allow_next_instance_of(Gitlab::GitalyClient::ServerService) do |instance| + allow(instance).to receive(:disk_statistics).and_return(response) + end + expect_next_instance_of(Gitlab::GitalyClient::ServerService) do |instance| + expect(instance).to receive(:storage_disk_statistics).and_return(storage_status) + end + end + + describe '.filesystem_disk_available' do + it 'returns disk available when the relevant storage status is found in the response' do + expect(storage_status).to receive(:available) + expect(described_class.filesystem_disk_available('default')).to eq(disk_available) + end + end + + describe '.filesystem_disk_used' do + it 'returns disk used when the relevant storage status is found in the response' do + expect(storage_status).to receive(:used) + expect(described_class.filesystem_disk_used('default')).to eq(disk_used) + end + end + end + describe '.stub_class' do it 'returns the gRPC health check stub' do expect(described_class.stub_class(:health_check)).to eq(::Grpc::Health::V1::Health::Stub) diff --git a/spec/lib/gitlab/import_export/import_failure_service_spec.rb b/spec/lib/gitlab/import_export/import_failure_service_spec.rb index 0351f88afdb..324328181e4 100644 --- a/spec/lib/gitlab/import_export/import_failure_service_spec.rb +++ b/spec/lib/gitlab/import_export/import_failure_service_spec.rb @@ -6,6 +6,7 @@ describe Gitlab::ImportExport::ImportFailureService do let(:importable) { create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') } let(:label) { create(:label) } let(:subject) { described_class.new(importable) } + let(:action) { "save_relation" } let(:relation_key) { "labels" } let(:relation_index) { 0 } @@ -15,7 +16,12 @@ describe Gitlab::ImportExport::ImportFailureService do let(:correlation_id) { 'my-correlation-id' } let(:retry_count) { 2 } let(:log_import_failure) do - subject.log_import_failure(relation_key, relation_index, exception, retry_count) + subject.log_import_failure( + source: action, + relation_key: relation_key, + relation_index: relation_index, + exception: exception, + retry_count: retry_count) end before do @@ -44,7 +50,7 @@ describe Gitlab::ImportExport::ImportFailureService do describe '#with_retry' do let(:perform_retry) do - subject.with_retry(relation_key, relation_index) do + subject.with_retry(action: action, relation_key: relation_key, relation_index: relation_index) do label.save! end end @@ -60,7 +66,12 @@ describe Gitlab::ImportExport::ImportFailureService do end it 'retries and logs import failure once with correct params' do - expect(subject).to receive(:log_import_failure).with(relation_key, relation_index, instance_of(exception), 1).once + expect(subject).to receive(:log_import_failure).with( + source: action, + relation_key: relation_key, + relation_index: relation_index, + exception: instance_of(exception), + retry_count: 1).once perform_retry end @@ -85,7 +96,11 @@ describe Gitlab::ImportExport::ImportFailureService do maximum_retry_count.times do |index| retry_count = index + 1 - expect(subject).to receive(:log_import_failure).with(relation_key, relation_index, instance_of(exception), retry_count) + expect(subject).to receive(:log_import_failure).with( + source: action, relation_key: relation_key, + relation_index: relation_index, + exception: instance_of(exception), + retry_count: retry_count) end expect { perform_retry }.to raise_exception(exception) diff --git a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb index ac9a63e8414..25f70420cda 100644 --- a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb @@ -498,6 +498,58 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do end end + context 'when post import action throw non-retriable exception' do + let(:exception) { StandardError.new('post_import_error') } + + before do + setup_import_export_config('light') + expect(project) + .to receive(:merge_requests) + .and_raise(exception) + end + + it 'report post import error' do + expect(restored_project_json).to eq(false) + expect(shared.errors).to include('post_import_error') + end + end + + context 'when post import action throw retriable exception one time' do + let(:exception) { GRPC::DeadlineExceeded.new } + + before do + setup_import_export_config('light') + expect(project) + .to receive(:merge_requests) + .and_raise(exception) + expect(project) + .to receive(:merge_requests) + .and_call_original + expect(restored_project_json).to eq(true) + end + + it_behaves_like 'restores project successfully', + issues: 1, + labels: 2, + label_with_priorities: 'A project label', + milestones: 1, + first_issue_labels: 1, + services: 1, + import_failures: 1 + + it 'records the failures in the database' do + import_failure = ImportFailure.last + + expect(import_failure.project_id).to eq(project.id) + expect(import_failure.relation_key).to be_nil + expect(import_failure.relation_index).to be_nil + expect(import_failure.exception_class).to eq('GRPC::DeadlineExceeded') + expect(import_failure.exception_message).to be_present + expect(import_failure.correlation_id_value).not_to be_empty + expect(import_failure.created_at).to be_present + end + end + context 'when the project has overridden params in import data' do before do setup_import_export_config('light') diff --git a/spec/lib/gitlab/sidekiq_config/worker_spec.rb b/spec/lib/gitlab/sidekiq_config/worker_spec.rb index f2fe51abd5e..ba6760f38b5 100644 --- a/spec/lib/gitlab/sidekiq_config/worker_spec.rb +++ b/spec/lib/gitlab/sidekiq_config/worker_spec.rb @@ -3,8 +3,11 @@ require 'fast_spec_helper' describe Gitlab::SidekiqConfig::Worker do - def worker_with_queue(queue) - described_class.new(double(queue: queue), ee: false) + def create_worker(queue:, weight: 0) + namespace = queue.include?(':') && queue.split(':').first + inner_worker = double(queue: queue, queue_namespace: namespace, get_weight: weight) + + described_class.new(inner_worker, ee: false) end describe '#ee?' do @@ -34,9 +37,9 @@ describe Gitlab::SidekiqConfig::Worker do describe 'delegations' do [ - :feature_category_not_owned?, :get_feature_category, + :feature_category_not_owned?, :get_feature_category, :get_weight, :get_worker_resource_boundary, :latency_sensitive_worker?, :queue, - :worker_has_external_dependencies? + :queue_namespace, :worker_has_external_dependencies? ].each do |meth| it "delegates #{meth} to the worker class" do worker = double @@ -50,8 +53,8 @@ describe Gitlab::SidekiqConfig::Worker do describe 'sorting' do it 'sorts queues with a namespace before those without a namespace' do - namespaced_worker = worker_with_queue('namespace:queue') - plain_worker = worker_with_queue('a_queue') + namespaced_worker = create_worker(queue: 'namespace:queue') + plain_worker = create_worker(queue: 'a_queue') expect([plain_worker, namespaced_worker].sort) .to eq([namespaced_worker, plain_worker]) @@ -59,12 +62,12 @@ describe Gitlab::SidekiqConfig::Worker do it 'sorts alphabetically by queue' do workers = [ - worker_with_queue('namespace:a'), - worker_with_queue('namespace:b'), - worker_with_queue('other_namespace:a'), - worker_with_queue('other_namespace:b'), - worker_with_queue('a'), - worker_with_queue('b') + create_worker(queue: 'namespace:a'), + create_worker(queue: 'namespace:b'), + create_worker(queue: 'other_namespace:a'), + create_worker(queue: 'other_namespace:b'), + create_worker(queue: 'a'), + create_worker(queue: 'b') ] expect(workers.shuffle.sort).to eq(workers) @@ -73,12 +76,26 @@ describe Gitlab::SidekiqConfig::Worker do describe 'YAML encoding' do it 'encodes the worker in YAML as a string of the queue' do - worker_a = worker_with_queue('a') - worker_b = worker_with_queue('b') + worker_a = create_worker(queue: 'a') + worker_b = create_worker(queue: 'b') expect(YAML.dump(worker_a)).to eq(YAML.dump('a')) expect(YAML.dump([worker_a, worker_b])) .to eq(YAML.dump(%w[a b])) end end + + describe '#namespace_and_weight' do + it 'returns a namespace, weight pair for the worker' do + expect(create_worker(queue: 'namespace:a', weight: 2).namespace_and_weight) + .to eq(['namespace', 2]) + end + end + + describe '#queue_and_weight' do + it 'returns a queue, weight pair for the worker' do + expect(create_worker(queue: 'namespace:a', weight: 2).queue_and_weight) + .to eq(['namespace:a', 2]) + end + end end diff --git a/spec/lib/gitlab/sidekiq_config_spec.rb b/spec/lib/gitlab/sidekiq_config_spec.rb index 39bb149cf73..20690a35dc8 100644 --- a/spec/lib/gitlab/sidekiq_config_spec.rb +++ b/spec/lib/gitlab/sidekiq_config_spec.rb @@ -80,4 +80,64 @@ describe Gitlab::SidekiqConfig do expect(described_class.all_queues_yml_outdated?).to be(false) end end + + describe '.queues_for_sidekiq_queues_yml' do + before do + workers = [ + Namespaces::RootStatisticsWorker, + Namespaces::ScheduleAggregationWorker, + MergeWorker, + ProcessCommitWorker + ].map { |worker| described_class::Worker.new(worker, ee: false) } + + allow(described_class).to receive(:workers).and_return(workers) + end + + it 'returns queues and weights, aggregating namespaces with the same weight' do + expected_queues = [ + ['merge', 5], + ['process_commit', 3], + ['update_namespace_statistics', 1] + ] + + expect(described_class.queues_for_sidekiq_queues_yml).to eq(expected_queues) + end + end + + describe '.sidekiq_queues_yml_outdated?' do + before do + workers = [ + Namespaces::RootStatisticsWorker, + Namespaces::ScheduleAggregationWorker, + MergeWorker, + ProcessCommitWorker + ].map { |worker| described_class::Worker.new(worker, ee: false) } + + allow(described_class).to receive(:workers).and_return(workers) + end + + let(:expected_queues) do + [ + ['merge', 5], + ['process_commit', 3], + ['update_namespace_statistics', 1] + ] + end + + it 'returns true if the YAML file does not match the application code' do + allow(File).to receive(:read) + .with(described_class::SIDEKIQ_QUEUES_PATH) + .and_return(YAML.dump(queues: expected_queues.reverse)) + + expect(described_class.sidekiq_queues_yml_outdated?).to be(true) + end + + it 'returns false if the YAML file matches the application code' do + allow(File).to receive(:read) + .with(described_class::SIDEKIQ_QUEUES_PATH) + .and_return(YAML.dump(queues: expected_queues)) + + expect(described_class.sidekiq_queues_yml_outdated?).to be(false) + end + end end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 38e15fc4582..4f729f2546c 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -2391,6 +2391,8 @@ describe Ci::Build do { key: 'GITLAB_CI', value: 'true', public: true, masked: false }, { key: 'CI_SERVER_URL', value: Gitlab.config.gitlab.url, public: true, masked: false }, { key: 'CI_SERVER_HOST', value: Gitlab.config.gitlab.host, public: true, masked: false }, + { key: 'CI_SERVER_PORT', value: Gitlab.config.gitlab.port.to_s, public: true, masked: false }, + { key: 'CI_SERVER_PROTOCOL', value: Gitlab.config.gitlab.protocol, public: true, masked: false }, { key: 'CI_SERVER_NAME', value: 'GitLab', public: true, masked: false }, { key: 'CI_SERVER_VERSION', value: Gitlab::VERSION, public: true, masked: false }, { key: 'CI_SERVER_VERSION_MAJOR', value: Gitlab.version_info.major.to_s, public: true, masked: false }, diff --git a/spec/requests/api/broadcast_messages_spec.rb b/spec/requests/api/broadcast_messages_spec.rb index 9dc639a25a2..a502d597da1 100644 --- a/spec/requests/api/broadcast_messages_spec.rb +++ b/spec/requests/api/broadcast_messages_spec.rb @@ -8,22 +8,10 @@ describe API::BroadcastMessages do set(:message) { create(:broadcast_message) } describe 'GET /broadcast_messages' do - it 'returns a 401 for anonymous users' do - get api('/broadcast_messages') - - expect(response).to have_gitlab_http_status(401) - end - - it 'returns a 403 for users' do - get api('/broadcast_messages', user) - - expect(response).to have_gitlab_http_status(403) - end - - it 'returns an Array of BroadcastMessages for admins' do + it 'returns an Array of BroadcastMessages' do create(:broadcast_message) - get api('/broadcast_messages', admin) + get api('/broadcast_messages') expect(response).to have_gitlab_http_status(200) expect(response).to include_pagination_headers @@ -34,21 +22,9 @@ describe API::BroadcastMessages do end describe 'GET /broadcast_messages/:id' do - it 'returns a 401 for anonymous users' do + it 'returns the specified message' do get api("/broadcast_messages/#{message.id}") - expect(response).to have_gitlab_http_status(401) - end - - it 'returns a 403 for users' do - get api("/broadcast_messages/#{message.id}", user) - - expect(response).to have_gitlab_http_status(403) - end - - it 'returns the specified message for admins' do - get api("/broadcast_messages/#{message.id}", admin) - expect(response).to have_gitlab_http_status(200) expect(json_response['id']).to eq message.id expect(json_response.keys) diff --git a/spec/requests/api/graphql/project/error_tracking/sentry_detailed_error_request_spec.rb b/spec/requests/api/graphql/project/error_tracking/sentry_detailed_error_request_spec.rb index 664206dec29..a1f9fa1f10c 100644 --- a/spec/requests/api/graphql/project/error_tracking/sentry_detailed_error_request_spec.rb +++ b/spec/requests/api/graphql/project/error_tracking/sentry_detailed_error_request_spec.rb @@ -57,6 +57,10 @@ describe 'getting a detailed sentry error' do expect(error_data['firstSeen']).to eql sentry_detailed_error.first_seen expect(error_data['lastSeen']).to eql sentry_detailed_error.last_seen expect(error_data['gitlabCommit']).to be nil + expect(error_data['externalBaseUrl']).to eq sentry_detailed_error.external_base_url + expect(error_data['gitlabIssuePath']).to eq sentry_detailed_error.gitlab_issue + expect(error_data['tags']['logger']).to eq sentry_detailed_error.tags[:logger] + expect(error_data['tags']['level']).to eq sentry_detailed_error.tags[:level] end it 'is expected to return the frequency correctly' do diff --git a/spec/requests/api/internal/base_spec.rb b/spec/requests/api/internal/base_spec.rb index 50754773fad..6f784830f9d 100644 --- a/spec/requests/api/internal/base_spec.rb +++ b/spec/requests/api/internal/base_spec.rb @@ -326,7 +326,7 @@ describe API::Internal::Base do expect(json_response["gitaly"]["repository"]["relative_path"]).to eq(project.repository.gitaly_repository.relative_path) expect(json_response["gitaly"]["address"]).to eq(Gitlab::GitalyClient.address(project.repository_storage)) expect(json_response["gitaly"]["token"]).to eq(Gitlab::GitalyClient.token(project.repository_storage)) - expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-inforef-uploadpack-cache' => 'true', 'gitaly-feature-get-tag-messages-go' => 'true', 'gitaly-feature-filter-shas-with-signatures-go' => 'true', 'gitaly-feature-cache-invalidator' => 'true', 'gitaly-feature-commit-without-batch-check' => 'true') + expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-inforef-uploadpack-cache' => 'true', 'gitaly-feature-filter-shas-with-signatures-go' => 'true', 'gitaly-feature-cache-invalidator' => 'true', 'gitaly-feature-commit-without-batch-check' => 'true') expect(user.reload.last_activity_on).to eql(Date.today) end end @@ -346,7 +346,7 @@ describe API::Internal::Base do expect(json_response["gitaly"]["repository"]["relative_path"]).to eq(project.repository.gitaly_repository.relative_path) expect(json_response["gitaly"]["address"]).to eq(Gitlab::GitalyClient.address(project.repository_storage)) expect(json_response["gitaly"]["token"]).to eq(Gitlab::GitalyClient.token(project.repository_storage)) - expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-inforef-uploadpack-cache' => 'true', 'gitaly-feature-get-tag-messages-go' => 'true', 'gitaly-feature-filter-shas-with-signatures-go' => 'true', 'gitaly-feature-cache-invalidator' => 'true', 'gitaly-feature-commit-without-batch-check' => 'true') + expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-inforef-uploadpack-cache' => 'true', 'gitaly-feature-filter-shas-with-signatures-go' => 'true', 'gitaly-feature-cache-invalidator' => 'true', 'gitaly-feature-commit-without-batch-check' => 'true') expect(user.reload.last_activity_on).to be_nil end end @@ -594,7 +594,7 @@ describe API::Internal::Base do expect(json_response["gitaly"]["repository"]["relative_path"]).to eq(project.repository.gitaly_repository.relative_path) expect(json_response["gitaly"]["address"]).to eq(Gitlab::GitalyClient.address(project.repository_storage)) expect(json_response["gitaly"]["token"]).to eq(Gitlab::GitalyClient.token(project.repository_storage)) - expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-inforef-uploadpack-cache' => 'true', 'gitaly-feature-get-tag-messages-go' => 'true', 'gitaly-feature-filter-shas-with-signatures-go' => 'true', 'gitaly-feature-cache-invalidator' => 'true', 'gitaly-feature-commit-without-batch-check' => 'true') + expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-inforef-uploadpack-cache' => 'true', 'gitaly-feature-filter-shas-with-signatures-go' => 'true', 'gitaly-feature-cache-invalidator' => 'true', 'gitaly-feature-commit-without-batch-check' => 'true') end end diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb index 42b4bd71b88..c3a5c0b0caa 100644 --- a/spec/requests/git_http_spec.rb +++ b/spec/requests/git_http_spec.rb @@ -108,7 +108,7 @@ describe 'Git HTTP requests' do shared_examples_for 'project path without .git suffix' do context "GET info/refs" do - let(:path) { "/#{project_path}/info/refs" } + let(:path) { "/#{repository_path}/info/refs" } context "when no params are added" do before do @@ -116,7 +116,7 @@ describe 'Git HTTP requests' do end it "redirects to the .git suffix version" do - expect(response).to redirect_to("/#{project_path}.git/info/refs") + expect(response).to redirect_to("/#{repository_path}.git/info/refs") end end @@ -128,7 +128,7 @@ describe 'Git HTTP requests' do end it "redirects to the .git suffix version" do - expect(response).to redirect_to("/#{project_path}.git/info/refs?service=#{params[:service]}") + expect(response).to redirect_to("/#{repository_path}.git/info/refs?service=#{params[:service]}") end end @@ -140,7 +140,7 @@ describe 'Git HTTP requests' do end it "redirects to the .git suffix version" do - expect(response).to redirect_to("/#{project_path}.git/info/refs?service=#{params[:service]}") + expect(response).to redirect_to("/#{repository_path}.git/info/refs?service=#{params[:service]}") end end @@ -159,13 +159,13 @@ describe 'Git HTTP requests' do context "POST git-upload-pack" do it "fails to find a route" do - expect { clone_post(project_path) }.to raise_error(ActionController::RoutingError) + expect { clone_post(repository_path) }.to raise_error(ActionController::RoutingError) end end context "POST git-receive-pack" do it "fails to find a route" do - expect { push_post(project_path) }.to raise_error(ActionController::RoutingError) + expect { push_post(repository_path) }.to raise_error(ActionController::RoutingError) end end end @@ -211,7 +211,7 @@ describe 'Git HTTP requests' do end it_behaves_like 'project path without .git suffix' do - let(:project_path) { "#{user.namespace.path}/project.git-project" } + let(:repository_path) { "#{user.namespace.path}/project.git-project" } end end end @@ -820,7 +820,7 @@ describe 'Git HTTP requests' do end it_behaves_like 'project path without .git suffix' do - let(:project_path) { create(:project, :repository, :public, path: 'project.git-project').full_path } + let(:repository_path) { create(:project, :repository, :public, path: 'project.git-project').full_path } end context "retrieving an info/refs file" do diff --git a/spec/support/shared_examples/lib/gitlab/import_export/import_failure_service_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/import_export/import_failure_service_shared_examples.rb index 55bd2401db1..801be5ae946 100644 --- a/spec/support/shared_examples/lib/gitlab/import_export/import_failure_service_shared_examples.rb +++ b/spec/support/shared_examples/lib/gitlab/import_export/import_failure_service_shared_examples.rb @@ -3,6 +3,7 @@ RSpec.shared_examples 'log import failure' do |importable_column| it 'tracks error' do extra = { + source: action, relation_key: relation_key, relation_index: relation_index, retry_count: retry_count @@ -11,7 +12,12 @@ RSpec.shared_examples 'log import failure' do |importable_column| expect(Gitlab::ErrorTracking).to receive(:track_exception).with(exception, extra) - subject.log_import_failure(relation_key, relation_index, exception, retry_count) + subject.log_import_failure( + source: action, + relation_key: relation_key, + relation_index: relation_index, + exception: exception, + retry_count: retry_count) end it 'saves data to ImportFailure' do @@ -21,6 +27,7 @@ RSpec.shared_examples 'log import failure' do |importable_column| aggregate_failures do expect(import_failure[importable_column]).to eq(importable.id) + expect(import_failure.source).to eq(action) expect(import_failure.relation_key).to eq(relation_key) expect(import_failure.relation_index).to eq(relation_index) expect(import_failure.exception_class).to eq('StandardError') |
