diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-01-21 14:21:10 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-01-21 14:21:10 +0000 |
| commit | cb0d23c455b73486fd1015f8ca9479b5b7e3585d (patch) | |
| tree | d7dc129a407fd74266d2dc561bebf24665197c2f /lib | |
| parent | c3e911be175c0aabfea1eb030f9e0ef23f5f3887 (diff) | |
| download | gitlab-ce-cb0d23c455b73486fd1015f8ca9479b5b7e3585d.tar.gz | |
Add latest changes from gitlab-org/gitlab@12-7-stable-ee
Diffstat (limited to 'lib')
244 files changed, 4359 insertions, 1359 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 56eccb036b6..1aee4fd30ee 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -43,6 +43,14 @@ module API header['X-Content-Type-Options'] = 'nosniff' end + before do + Gitlab::ApplicationContext.push( + user: -> { current_user }, + project: -> { @project }, + namespace: -> { @group } + ) + end + # The locale is set to the current user's locale when `current_user` is loaded after { Gitlab::I18n.use_default_locale } @@ -96,6 +104,7 @@ module API # Keep in alphabetical order mount ::API::AccessRequests + mount ::API::Appearance mount ::API::Applications mount ::API::Avatar mount ::API::AwardEmoji @@ -108,6 +117,7 @@ module API mount ::API::DeployKeys mount ::API::Deployments mount ::API::Environments + mount ::API::ErrorTracking mount ::API::Events mount ::API::Features mount ::API::Files diff --git a/lib/api/appearance.rb b/lib/api/appearance.rb new file mode 100644 index 00000000000..a775102e87d --- /dev/null +++ b/lib/api/appearance.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module API + class Appearance < Grape::API + before { authenticated_as_admin! } + + helpers do + def current_appearance + @current_appearance ||= (::Appearance.current || ::Appearance.new) + end + end + + desc 'Get the current appearance' do + success Entities::Appearance + end + get "application/appearance" do + present current_appearance, with: Entities::Appearance + end + + desc 'Modify appearance' do + success Entities::Appearance + end + params do + optional :title, type: String, desc: 'Instance title on the sign in / sign up page' + optional :description, type: String, desc: 'Markdown text shown on the sign in / sign up page' + # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 + optional :logo, type: File, desc: 'Instance image used on the sign in / sign up page' # rubocop:disable Scalability/FileUploads + optional :header_logo, type: File, desc: 'Instance image used for the main navigation bar' # rubocop:disable Scalability/FileUploads + optional :favicon, type: File, desc: 'Instance favicon in .ico/.png format' # rubocop:disable Scalability/FileUploads + optional :new_project_guidelines, type: String, desc: 'Markmarkdown text shown on the new project page' + optional :header_message, type: String, desc: 'Message within the system header bar' + optional :footer_message, type: String, desc: 'Message within the system footer bar' + optional :message_background_color, type: String, desc: 'Background color for the system header / footer bar' + optional :message_font_color, type: String, desc: 'Font color for the system header / footer bar' + optional :email_header_and_footer_enabled, type: Boolean, desc: 'Add header and footer to all outgoing emails if enabled' + end + put "application/appearance" do + attrs = declared_params(include_missing: false) + + if current_appearance.update(attrs) + present current_appearance, with: Entities::Appearance + else + render_validation_error!(current_appearance) + end + end + end +end diff --git a/lib/api/applications.rb b/lib/api/applications.rb index 92717e04543..4e9843e17e8 100644 --- a/lib/api/applications.rb +++ b/lib/api/applications.rb @@ -38,7 +38,7 @@ module API application = ApplicationsFinder.new(params).execute application.destroy - status 204 + no_content! end end end diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 89b7e5c5e4b..7a815fa3dde 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -27,7 +27,6 @@ module API ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" ].each do |endpoint| - desc 'Get a list of project +awardable+ award emoji' do detail 'This feature was introduced in 8.9' success Entities::AwardEmoji diff --git a/lib/api/badges.rb b/lib/api/badges.rb index e987c24c707..d2152fad07b 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -135,7 +135,6 @@ module API end destroy_conditionally!(badge) - body false end end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index ce3ee0d7e61..999bf1627c1 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -57,7 +57,7 @@ module API requires :branch, type: String, desc: 'The name of the branch' end head do - user_project.repository.branch_exists?(params[:branch]) ? status(204) : status(404) + user_project.repository.branch_exists?(params[:branch]) ? no_content! : not_found! end get do branch = find_branch!(params[:branch]) diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index d108c811f4b..6e26ee309f0 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -71,27 +71,27 @@ module API ref = params[:ref] ref ||= pipeline&.ref - ref ||= @project.repository.branch_names_contains(commit.sha).first + ref ||= user_project.repository.branch_names_contains(commit.sha).first not_found! 'References for commit' unless ref name = params[:name] || params[:context] || 'default' unless pipeline - pipeline = @project.ci_pipelines.create!( + pipeline = user_project.ci_pipelines.create!( source: :external, sha: commit.sha, ref: ref, user: current_user, - protected: @project.protected_for?(ref)) + protected: user_project.protected_for?(ref)) end status = GenericCommitStatus.running_or_pending.find_or_initialize_by( - project: @project, + project: user_project, pipeline: pipeline, name: name, ref: ref, user: current_user, - protected: @project.protected_for?(ref) + protected: user_project.protected_for?(ref) ) optional_attributes = @@ -117,7 +117,7 @@ module API render_api_error!('invalid state', 400) end - MergeRequest.where(source_project: @project, source_branch: ref) + MergeRequest.where(source_project: user_project, source_branch: ref) .update_all(head_pipeline_id: pipeline.id) if pipeline.latest? present status, with: Entities::CommitStatus diff --git a/lib/api/custom_attributes_endpoints.rb b/lib/api/custom_attributes_endpoints.rb index 2149e04451e..ef1264126f4 100644 --- a/lib/api/custom_attributes_endpoints.rb +++ b/lib/api/custom_attributes_endpoints.rb @@ -77,7 +77,7 @@ module API resource.custom_attributes.find_by!(key: params[:key]).destroy - status 204 + no_content! end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 84d1d8a0aac..487d4e37a56 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -21,6 +21,14 @@ module API optional :sort, type: String, values: DeploymentsFinder::ALLOWED_SORT_DIRECTIONS, default: DeploymentsFinder::DEFAULT_SORT_DIRECTION, desc: 'Sort by asc (ascending) or desc (descending)' optional :updated_after, type: DateTime, desc: 'Return deployments updated after the specified date' optional :updated_before, type: DateTime, desc: 'Return deployments updated before the specified date' + optional :environment, + type: String, + desc: 'The name of the environment to filter deployments by' + + optional :status, + type: String, + values: Deployment.statuses.keys, + desc: 'The status to filter deployments by' end get ':id/deployments' do @@ -127,6 +135,26 @@ module API render_validation_error!(deployment) end end + + helpers Helpers::MergeRequestsHelpers + + desc 'Get all merge requests of a deployment' do + detail 'This feature was introduced in GitLab 12.7.' + success Entities::MergeRequestBasic + end + params do + requires :deployment_id, type: Integer, desc: 'The deployment ID' + use :merge_requests_base_params + end + + get ':id/deployments/:deployment_id/merge_requests' do + authorize! :read_deployment, user_project + + mr_params = declared_params.merge(deployment_id: params[:deployment_id]) + merge_requests = MergeRequestsFinder.new(current_user, mr_params).execute + + present merge_requests, { with: Entities::MergeRequestBasic, current_user: current_user } + end end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 76963777566..dfd0e676586 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -178,6 +178,15 @@ module API expose :only_protected_branches end + class ContainerExpirationPolicy < Grape::Entity + expose :cadence + expose :enabled + expose :keep_n + expose :older_than + expose :name_regex + expose :next_run_at + end + class ProjectImportStatus < ProjectIdentity expose :import_status @@ -276,6 +285,8 @@ module API expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :resolve_outdated_diff_discussions expose :container_registry_enabled + expose :container_expiration_policy, using: Entities::ContainerExpirationPolicy, + if: -> (project, _) { project.container_expiration_policy } # Expose old field names with the new permissions methods to keep API compatible # TODO: remove in API v5, replaced by *_access_level @@ -324,6 +335,7 @@ module API expose :remove_source_branch_after_merge expose :printing_merge_request_link_enabled expose :merge_method + expose :suggestion_commit_message expose :statistics, using: 'API::Entities::ProjectStatistics', if: -> (project, options) { options[:statistics] && Ability.allowed?(options[:current_user], :read_statistics, project) } @@ -331,6 +343,7 @@ module API expose :auto_devops_deploy_strategy do |project, options| project.auto_devops.nil? ? 'continuous' : project.auto_devops.deploy_strategy end + expose :autoclose_referenced_issues # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) @@ -340,6 +353,7 @@ module API # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20555 super(projects_relation).preload(:group) .preload(:ci_cd_settings) + .preload(:container_expiration_policy) .preload(:auto_devops) .preload(project_group_links: { group: :route }, fork_network: :root_project, @@ -400,6 +414,7 @@ module API expose :auto_devops_enabled expose :subgroup_creation_level_str, as: :subgroup_creation_level expose :emails_disabled + expose :mentions_disabled expose :lfs_enabled?, as: :lfs_enabled expose :avatar_url do |group, options| group.avatar_url(only_path: false) @@ -569,6 +584,20 @@ module API end end + class IssuableReferences < Grape::Entity + expose :short do |issuable| + issuable.to_reference + end + + expose :relative do |issuable, options| + issuable.to_reference(options[:group] || options[:project]) + end + + expose :full do |issuable| + issuable.to_reference(full: true) + end + end + class Diff < Grape::Entity expose :old_path, :new_path, :a_mode, :b_mode expose :new_file?, as: :new_file @@ -585,6 +614,7 @@ module API end class ProtectedBranch < Grape::Entity + expose :id expose :name expose :push_access_levels, using: Entities::ProtectedRefAccess expose :merge_access_levels, using: Entities::ProtectedRefAccess @@ -676,6 +706,10 @@ module API end end + expose :references, with: IssuableReferences do |issue| + issue + end + # Calculating the value of subscribed field triggers Markdown # processing. We can't do that for multiple issues / merge # requests in a single API request. @@ -761,9 +795,12 @@ module API expose :author, :assignees, using: Entities::UserBasic expose :source_project_id, :target_project_id - expose :labels do |merge_request| - # Avoids an N+1 query since labels are preloaded - merge_request.labels.map(&:title).sort + expose :labels do |merge_request, options| + if options[:with_labels_details] + ::API::Entities::LabelBasic.represent(merge_request.labels.sort_by(&:title)) + else + merge_request.labels.map(&:title).sort + end end expose :work_in_progress?, as: :work_in_progress expose :milestone, using: Entities::Milestone @@ -787,10 +824,16 @@ module API # Deprecated expose :allow_collaboration, as: :allow_maintainer_to_push, if: -> (merge_request, _) { merge_request.for_fork? } + # reference is deprecated in favour of references + # Introduced [Gitlab 12.6](https://gitlab.com/gitlab-org/gitlab/merge_requests/20354) expose :reference do |merge_request, options| merge_request.to_reference(options[:project]) end + expose :references, with: IssuableReferences do |merge_request| + merge_request + end + expose :web_url do |merge_request| Gitlab::UrlBuilder.build(merge_request) end @@ -883,6 +926,10 @@ module API expose :user, using: Entities::UserPublic end + class DeployKeyWithUser < SSHKeyWithUser + expose :deploy_keys_projects + end + class DeployKeysProject < Grape::Entity expose :deploy_key, merge: true, using: Entities::SSHKey expose :can_push @@ -1082,12 +1129,19 @@ module API end end - class ProjectService < Grape::Entity - expose :id, :title, :created_at, :updated_at, :active + class ProjectServiceBasic < Grape::Entity + expose :id, :title + expose :slug do |service| + service.to_param.dasherize + end + expose :created_at, :updated_at, :active expose :commit_events, :push_events, :issues_events, :confidential_issues_events expose :merge_requests_events, :tag_push_events, :note_events expose :confidential_note_events, :pipeline_events, :wiki_page_events - expose :job_events + expose :job_events, :comment_on_event_enabled + end + + class ProjectService < ProjectServiceBasic # Expose serialized properties expose :properties do |service, options| # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 @@ -1142,7 +1196,7 @@ module API end class LabelBasic < Grape::Entity - expose :id, :name, :color, :description, :text_color + expose :id, :name, :color, :description, :description_html, :text_color end class Label < LabelBasic @@ -1300,6 +1354,30 @@ module API expose :allow_local_requests_from_web_hooks_and_services, as: :allow_local_requests_from_hooks_and_services end + class Appearance < Grape::Entity + expose :title + expose :description + + expose :logo do |appearance, options| + appearance.logo.url + end + + expose :header_logo do |appearance, options| + appearance.header_logo.url + end + + expose :favicon do |appearance, options| + appearance.favicon.url + end + + expose :new_project_guidelines + expose :header_message + expose :footer_message + expose :message_background_color + expose :message_font_color + expose :email_header_and_footer_enabled + end + # deprecated old Release representation class TagRelease < Grape::Entity expose :tag, as: :tag_name diff --git a/lib/api/entities/error_tracking.rb b/lib/api/entities/error_tracking.rb new file mode 100644 index 00000000000..c762c274486 --- /dev/null +++ b/lib/api/entities/error_tracking.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module ErrorTracking + class ProjectSetting < Grape::Entity + expose :enabled, as: :active + expose :project_name + expose :sentry_external_url + expose :api_url + end + end + end +end diff --git a/lib/api/error_tracking.rb b/lib/api/error_tracking.rb new file mode 100644 index 00000000000..f92f1326daa --- /dev/null +++ b/lib/api/error_tracking.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module API + class ErrorTracking < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get error tracking settings for the project' do + detail 'This feature was introduced in GitLab 12.7.' + success Entities::ErrorTracking::ProjectSetting + end + + get ':id/error_tracking/settings' do + authorize! :admin_operations, user_project + + setting = user_project.error_tracking_setting + + not_found!('Error Tracking Setting') unless setting + + present setting, with: Entities::ErrorTracking::ProjectSetting + end + end + end +end diff --git a/lib/api/features.rb b/lib/api/features.rb index 4dc1834c644..69b751e9bdb 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -74,7 +74,7 @@ module API delete ':name' do Feature.get(params[:name]).remove - status 204 + no_content! end end end diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index eae29f5b5dd..9e9f5101285 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -67,7 +67,7 @@ module API milestone = user_group.milestones.find(params[:milestone_id]) Milestones::DestroyService.new(user_group, current_user).execute(milestone) - status(204) + no_content! end desc 'Get all issues for a single group milestone' do diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 37cb6d6a639..b2f5def4048 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -4,6 +4,7 @@ module API module Helpers include Gitlab::Utils include Helpers::Pagination + include Helpers::PaginationStrategies SUDO_HEADER = "HTTP_SUDO" GITLAB_SHARED_SECRET_HEADER = "Gitlab-Shared-Secret" @@ -30,6 +31,7 @@ module API check_unmodified_since!(last_updated) status 204 + body false if block_given? yield resource @@ -363,6 +365,10 @@ module API render_api_error!('204 No Content', 204) end + def created! + render_api_error!('201 Created', 201) + end + def accepted! render_api_error!('202 Accepted', 202) end diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb index 2cc18acb7ec..e0fea4c7c96 100644 --- a/lib/api/helpers/groups_helpers.rb +++ b/lib/api/helpers/groups_helpers.rb @@ -18,6 +18,7 @@ module API optional :auto_devops_enabled, type: Boolean, desc: 'Default to Auto DevOps pipeline for all projects within this group' optional :subgroup_creation_level, type: String, values: ::Gitlab::Access.subgroup_creation_string_values, desc: 'Allowed to create subgroups', as: :subgroup_creation_level_str optional :emails_disabled, type: Boolean, desc: 'Disable email notifications' + optional :mentions_disabled, type: Boolean, desc: 'Disable a group from getting mentioned' optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index b03eb5ad440..cc4a0d348a0 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -52,7 +52,7 @@ module API def log_user_activity(actor) commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS - ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action]) + ::Users::ActivityService.new(actor).execute if commands.include?(params[:action]) end def merge_request_urls @@ -107,8 +107,10 @@ module API if params[:gl_repository] @project, @repo_type = Gitlab::GlRepository.parse(params[:gl_repository]) @redirected_path = nil - else + elsif params[:project] @project, @repo_type, @redirected_path = Gitlab::RepoPath.parse(params[:project]) + else + @project, @repo_type, @redirected_path = nil, nil, nil end end # rubocop:enable Gitlab/ModuleWithInstanceVariables @@ -120,21 +122,13 @@ module API end def gl_project_path - if wiki? - project.wiki.full_path - else - project.full_path - end + repository.full_path end # Return the repository depending on whether we want the wiki or the # regular repository def repository - if repo_type.wiki? - project.wiki.repository - else - project.repository - end + @repository ||= repo_type.repository_for(project) end # Return the Gitaly Address if it is enabled diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 9e624903a62..d06c59907b4 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -5,6 +5,11 @@ module API module Helpers module MembersHelpers + extend Grape::API::Helpers + + params :optional_filter_params_ee do + end + def find_source(source_type, id) public_send("find_#{source_type}!", id) # rubocop:disable GitlabSecurity/PublicSend end @@ -36,9 +41,15 @@ module API GroupMembersFinder.new(group).execute end + def create_member(current_user, user, source, params) + source.add_user(user, params[:access_level], current_user: current_user, expires_at: params[:expires_at]) + end + def present_members(members) present members, with: Entities::Member, current_user: current_user end end end end + +API::Helpers::MembersHelpers.prepend_if_ee('EE::API::Helpers::MembersHelpers') diff --git a/lib/api/helpers/merge_requests_helpers.rb b/lib/api/helpers/merge_requests_helpers.rb new file mode 100644 index 00000000000..0126d7a3756 --- /dev/null +++ b/lib/api/helpers/merge_requests_helpers.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module API + module Helpers + module MergeRequestsHelpers + extend Grape::API::Helpers + include ::API::Helpers::CustomValidators + + params :merge_requests_base_params do + optional :state, + type: String, + values: %w[opened closed locked merged all], + default: 'all', + desc: 'Return opened, closed, locked, merged, or all merge requests' + optional :order_by, + type: String, + values: %w[created_at updated_at], + default: 'created_at', + desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' + optional :sort, + type: String, + values: %w[asc desc], + default: 'desc', + desc: 'Return merge requests sorted in `asc` or `desc` order.' + optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' + optional :labels, + type: Array[String], + coerce_with: Validations::Types::LabelsList.coerce, + desc: 'Comma-separated list of label names' + optional :with_labels_details, type: Boolean, desc: 'Return titles of labels and other details', default: false + optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' + optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' + optional :updated_after, type: DateTime, desc: 'Return merge requests updated after the specified time' + optional :updated_before, type: DateTime, desc: 'Return merge requests updated before the specified time' + optional :view, + type: String, + values: %w[simple], + desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' + optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' + optional :assignee_id, + types: [Integer, String], + integer_none_any: true, + desc: 'Return merge requests which are assigned to the user with the given ID' + optional :scope, + type: String, + values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], + desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`' + optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' + optional :source_branch, type: String, desc: 'Return merge requests with the given source branch' + optional :source_project_id, type: Integer, desc: 'Return merge requests with the given source project id' + optional :target_branch, type: String, desc: 'Return merge requests with the given target branch' + optional :search, + type: String, + desc: 'Search merge requests for text present in the title, description, or any combination of these' + optional :in, type: String, desc: '`title`, `description`, or a string joining them with comma' + optional :wip, type: String, values: %w[yes no], desc: 'Search merge requests for WIP in the title' + end + + params :optional_scope_param do + optional :scope, + type: String, + values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], + default: 'created_by_me', + desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`' + end + end + end +end diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 1b63e450a12..a6ae9a87f98 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -3,34 +3,9 @@ module API module Helpers module Pagination - # This returns an ActiveRecord relation def paginate(relation) Gitlab::Pagination::OffsetPagination.new(self).paginate(relation) end - - # This applies pagination and executes the query - # It always returns an array instead of an ActiveRecord relation - def paginate_and_retrieve!(relation) - offset_or_keyset_pagination(relation).to_a - end - - private - - def offset_or_keyset_pagination(relation) - return paginate(relation) unless keyset_pagination_enabled? - - request_context = Gitlab::Pagination::Keyset::RequestContext.new(self) - - unless Gitlab::Pagination::Keyset.available?(request_context, relation) - return error!('Keyset pagination is not yet available for this type of request', 405) - end - - Gitlab::Pagination::Keyset.paginate(request_context, relation) - end - - def keyset_pagination_enabled? - params[:pagination] == 'keyset' && Feature.enabled?(:api_keyset_pagination, default_enabled: true) - end end end end diff --git a/lib/api/helpers/pagination_strategies.rb b/lib/api/helpers/pagination_strategies.rb new file mode 100644 index 00000000000..5f63635297a --- /dev/null +++ b/lib/api/helpers/pagination_strategies.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module API + module Helpers + module PaginationStrategies + def paginate_with_strategies(relation) + paginator = paginator(relation) + + yield(paginator.paginate(relation)).tap do |records, _| + paginator.finalize(records) + end + end + + def paginator(relation) + return Gitlab::Pagination::OffsetPagination.new(self) unless keyset_pagination_enabled? + + request_context = Gitlab::Pagination::Keyset::RequestContext.new(self) + + unless Gitlab::Pagination::Keyset.available?(request_context, relation) + return error!('Keyset pagination is not yet available for this type of request', 405) + end + + Gitlab::Pagination::Keyset::Pager.new(request_context) + end + + private + + def keyset_pagination_enabled? + params[:pagination] == 'keyset' && Feature.enabled?(:api_keyset_pagination, default_enabled: true) + end + end + end +end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 47b1f037eb8..6333e00daf5 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -32,6 +32,9 @@ module API optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' optional :remove_source_branch_after_merge, type: Boolean, desc: 'Remove the source branch by default after merge' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' + optional :container_expiration_policy_attributes, type: Hash do + use :optional_container_expiration_policy_params + end optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' optional :public_builds, type: Boolean, desc: 'Perform public builds' @@ -43,10 +46,12 @@ module API optional :avatar, type: File, desc: 'Avatar image for project' # rubocop:disable Scalability/FileUploads optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' + optional :suggestion_commit_message, type: String, desc: 'The commit message used to apply merge request suggestions' optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" optional :ci_default_git_depth, type: Integer, desc: 'Default number of revisions for shallow cloning' optional :auto_devops_enabled, type: Boolean, desc: 'Flag indication if Auto DevOps is enabled' optional :auto_devops_deploy_strategy, type: String, values: %w(continuous manual timed_incremental), desc: 'Auto Deploy strategy' + optional :autoclose_referenced_issues, type: Boolean, desc: 'Flag indication if referenced issues auto-closing is enabled' end params :optional_project_params_ee do @@ -71,6 +76,14 @@ module API params :optional_update_params_ee do end + params :optional_container_expiration_policy_params do + optional :cadence, type: String, desc: 'Container expiration policy cadence for recurring job' + optional :keep_n, type: String, desc: 'Container expiration policy number of images to keep' + optional :older_than, type: String, desc: 'Container expiration policy remove images older than value' + optional :name_regex, type: String, desc: 'Container expiration policy regex for image removal' + optional :enabled, type: Boolean, desc: 'Flag indication if container expiration policy is enabled' + end + def self.update_params_at_least_one_of [ :auto_devops_enabled, @@ -83,8 +96,10 @@ module API :ci_config_path, :ci_default_git_depth, :container_registry_enabled, + :container_expiration_policy_attributes, :default_branch, :description, + :autoclose_referenced_issues, :issues_access_level, :lfs_enabled, :merge_requests_access_level, @@ -105,6 +120,7 @@ module API :visibility, :wiki_access_level, :avatar, + :suggestion_commit_message, # TODO: remove in API v5, replaced by *_access_level :issues_enabled, diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb index b77be6edcf7..c02244c7202 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/services_helpers.rb @@ -365,6 +365,12 @@ module API name: :send_from_committer_email, type: Boolean, desc: 'Send from committer' + }, + { + required: false, + name: :branches_to_be_notified, + type: String, + desc: 'Branches for which notifications are to be sent' } ], 'external-wiki' => [ diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 50142b8641e..d64de2bb465 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -6,6 +6,13 @@ module API class Base < Grape::API before { authenticate_by_gitlab_shell_token! } + before do + Gitlab::ApplicationContext.push( + user: -> { actor&.user }, + project: -> { project } + ) + end + helpers ::API::Helpers::InternalHelpers UNKNOWN_CHECK_RESULT_ERROR = 'Unknown check result'.freeze @@ -205,7 +212,12 @@ module API status 200 response = Gitlab::InternalPostReceive::Response.new + + # Try to load the project and users so we have the application context + # available for logging before we schedule any jobs. user = actor.user + project + push_options = Gitlab::PushOptions.new(params[:push_options]) response.reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease @@ -224,9 +236,9 @@ module API response.add_merge_request_urls(merge_request_urls) - # A user is not guaranteed to be returned; an orphaned write deploy + # Neither User nor Project are guaranteed to be returned; an orphaned write deploy # key could be used - if user + if user && project redirect_message = Gitlab::Checks::ProjectMoved.fetch_message(user.id, project.id) project_created_message = Gitlab::Checks::ProjectCreated.fetch_message(user.id, project.id) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4208385a48d..4e21815fa35 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -48,7 +48,7 @@ module API end params :issues_params do - optional :with_labels_details, type: Boolean, desc: 'Return more label data than just lable title', default: false + optional :with_labels_details, type: Boolean, desc: 'Return titles of labels and other details', default: false optional :state, type: String, values: %w[opened closed all], default: 'all', desc: 'Return opened, closed, or all issues' optional :order_by, type: String, values: Helpers::IssuesHelpers.sort_options, default: 'created_at', @@ -122,16 +122,15 @@ module API use :issues_params end get ":id/issues" do - group = find_group!(params[:id]) - - issues = paginate(find_issues(group_id: group.id, include_subgroups: true)) + issues = paginate(find_issues(group_id: user_group.id, include_subgroups: true)) options = { with: Entities::Issue, with_labels_details: declared_params[:with_labels_details], current_user: current_user, issuable_metadata: issuable_meta_data(issues, 'Issue', current_user), - include_subscribed: false + include_subscribed: false, + group: user_group } present issues, options @@ -142,9 +141,7 @@ module API use :issues_stats_params end get ":id/issues_statistics" do - group = find_group!(params[:id]) - - present issues_statistics(group_id: group.id, include_subgroups: true), with: Grape::Presenters::Presenter + present issues_statistics(group_id: user_group.id, include_subgroups: true), with: Grape::Presenters::Presenter end end @@ -161,9 +158,7 @@ module API use :issues_params end get ":id/issues" do - project = find_project!(params[:id]) - - issues = paginate(find_issues(project_id: project.id)) + issues = paginate(find_issues(project_id: user_project.id)) options = { with: Entities::Issue, @@ -182,9 +177,7 @@ module API use :issues_stats_params end get ":id/issues_statistics" do - project = find_project!(params[:id]) - - present issues_statistics(project_id: project.id), with: Grape::Presenters::Presenter + present issues_statistics(project_id: user_project.id), with: Grape::Presenters::Presenter end desc 'Get a single project issue' do @@ -227,18 +220,22 @@ module API issue_params = convert_parameters_from_legacy_format(issue_params) - issue = ::Issues::CreateService.new(user_project, - current_user, - issue_params.merge(request: request, api: true)).execute - - if issue.spam? - render_api_error!({ error: 'Spam detected' }, 400) - end - - if issue.valid? - present issue, with: Entities::Issue, current_user: current_user, project: user_project - else - render_validation_error!(issue) + begin + issue = ::Issues::CreateService.new(user_project, + current_user, + issue_params.merge(request: request, api: true)).execute + + if issue.spam? + render_api_error!({ error: 'Spam detected' }, 400) + end + + if issue.valid? + present issue, with: Entities::Issue, current_user: current_user, project: user_project + else + render_validation_error!(issue) + end + rescue ::ActiveRecord::RecordNotUnique + render_api_error!('Duplicated issue', 409) end end diff --git a/lib/api/keys.rb b/lib/api/keys.rb index 8f837107192..bec3dc9bd97 100644 --- a/lib/api/keys.rb +++ b/lib/api/keys.rb @@ -26,12 +26,15 @@ module API get do authenticated_with_can_read_all_resources! - finder_params = params.merge(key_type: 'ssh') - - key = KeysFinder.new(current_user, finder_params).execute + key = KeysFinder.new(current_user, params).execute not_found!('Key') unless key - present key, with: Entities::SSHKeyWithUser, current_user: current_user + + if key.type == "DeployKey" + present key, with: Entities::DeployKeyWithUser, current_user: current_user + else + present key, with: Entities::SSHKeyWithUser, current_user: current_user + end rescue KeysFinder::InvalidFingerprint render_api_error!('Failed to return the key', 400) end diff --git a/lib/api/members.rb b/lib/api/members.rb index 3526671e7f9..e4df2f341c6 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -19,6 +19,7 @@ module API params do optional :query, type: String, desc: 'A query string to search for members' optional :user_ids, type: Array[Integer], desc: 'Array of user ids to look up for membership' + use :optional_filter_params_ee use :pagination end @@ -100,12 +101,12 @@ module API user = User.find_by_id(params[:user_id]) not_found!('User') unless user - member = source.add_user(user, params[:access_level], current_user: current_user, expires_at: params[:expires_at]) + member = create_member(current_user, user, source, params) if !member not_allowed! # This currently can only be reached in EE elsif member.persisted? && member.valid? - present_members member + present_members(member) else render_validation_error!(member) end @@ -157,5 +158,3 @@ module API end end end - -API::Members.prepend_if_ee('EE::API::Members') diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 794237f8032..bd857278ee5 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -7,6 +7,7 @@ module API before { authenticate_non_get! } helpers ::Gitlab::IssuableMetadata + helpers Helpers::MergeRequestsHelpers # EE::API::MergeRequests would override the following helpers helpers do @@ -68,12 +69,21 @@ module API end end - def not_automatically_mergeable?(merge_when_pipeline_succeeds, merge_request) - merge_when_pipeline_succeeds && !merge_request.head_pipeline_active? && !merge_request.actual_head_pipeline_success? + def automatically_mergeable?(merge_when_pipeline_succeeds, merge_request) + pipeline_active = merge_request.head_pipeline_active? || merge_request.actual_head_pipeline_active? + merge_when_pipeline_succeeds && merge_request.mergeable_state?(skip_ci_check: true) && pipeline_active + end + + def immediately_mergeable?(merge_when_pipeline_succeeds, merge_request) + if merge_when_pipeline_succeeds + merge_request.actual_head_pipeline_success? + else + merge_request.mergeable_state? + end end def serializer_options_for(merge_requests) - options = { with: Entities::MergeRequestBasic, current_user: current_user } + options = { with: Entities::MergeRequestBasic, current_user: current_user, with_labels_details: declared_params[:with_labels_details] } if params[:view] == 'simple' options[:with] = Entities::MergeRequestSimple @@ -98,32 +108,7 @@ module API end params :merge_requests_params do - optional :state, type: String, values: %w[opened closed locked merged all], default: 'all', - desc: 'Return opened, closed, locked, merged, or all merge requests' - optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', - desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return merge requests sorted in `asc` or `desc` order.' - optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' - optional :labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names' - optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' - optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' - optional :updated_after, type: DateTime, desc: 'Return merge requests updated after the specified time' - optional :updated_before, type: DateTime, desc: 'Return merge requests updated before the specified time' - optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' - optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' - optional :assignee_id, types: [Integer, String], integer_none_any: true, - desc: 'Return merge requests which are assigned to the user with the given ID' - optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], - desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`' - optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' - optional :source_branch, type: String, desc: 'Return merge requests with the given source branch' - optional :source_project_id, type: Integer, desc: 'Return merge requests with the given source project id' - optional :target_branch, type: String, desc: 'Return merge requests with the given target branch' - optional :search, type: String, desc: 'Search merge requests for text present in the title, description, or any combination of these' - optional :in, type: String, desc: '`title`, `description`, or a string joining them with comma' - optional :wip, type: String, values: %w[yes no], desc: 'Search merge requests for WIP in the title' - + use :merge_requests_base_params use :optional_merge_requests_search_params use :pagination end @@ -135,8 +120,7 @@ module API end params do use :merge_requests_params - optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], default: 'created_by_me', - desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`' + use :optional_scope_param end get do authenticate! unless params[:scope] == 'all' @@ -157,11 +141,9 @@ module API use :merge_requests_params end get ":id/merge_requests" do - group = find_group!(params[:id]) - - merge_requests = find_merge_requests(group_id: group.id, include_subgroups: true) + merge_requests = find_merge_requests(group_id: user_group.id, include_subgroups: true) - present merge_requests, serializer_options_for(merge_requests) + present merge_requests, serializer_options_for(merge_requests).merge(group: user_group) end end @@ -215,7 +197,7 @@ module API merge_requests = find_merge_requests(project_id: user_project.id) - options = serializer_options_for(merge_requests) + options = serializer_options_for(merge_requests).merge(project: user_project) options[:project] = user_project present merge_requests, options @@ -394,16 +376,18 @@ module API Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42317') merge_request = find_project_merge_request(params[:merge_request_iid]) - merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds]) - not_automatically_mergeable = not_automatically_mergeable?(merge_when_pipeline_succeeds, merge_request) # Merge request can not be merged # because user dont have permissions to push into target branch unauthorized! unless merge_request.can_be_merged_by?(current_user) - not_allowed! if !merge_request.mergeable_state?(skip_ci_check: merge_when_pipeline_succeeds) || not_automatically_mergeable + merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds]) + automatically_mergeable = automatically_mergeable?(merge_when_pipeline_succeeds, merge_request) + immediately_mergeable = immediately_mergeable?(merge_when_pipeline_succeeds, merge_request) - render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?(skip_ci_check: merge_when_pipeline_succeeds) + not_allowed! if !immediately_mergeable && !automatically_mergeable + + render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?(skip_ci_check: automatically_mergeable) check_sha_param!(params, merge_request) @@ -416,13 +400,13 @@ module API sha: params[:sha] || merge_request.diff_head_sha ) - if merge_when_pipeline_succeeds - AutoMergeService.new(merge_request.target_project, current_user, merge_params) - .execute(merge_request, AutoMergeService::STRATEGY_MERGE_WHEN_PIPELINE_SUCCEEDS) - else + if immediately_mergeable ::MergeRequests::MergeService .new(merge_request.target_project, current_user, merge_params) .execute(merge_request) + elsif automatically_mergeable + AutoMergeService.new(merge_request.target_project, current_user, merge_params) + .execute(merge_request, AutoMergeService::STRATEGY_MERGE_WHEN_PIPELINE_SUCCEEDS) end present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project @@ -455,12 +439,15 @@ module API desc 'Rebase the merge request against its target branch' do detail 'This feature was added in GitLab 11.6' end + params do + optional :skip_ci, type: Boolean, desc: 'Do not create CI pipeline' + end put ':id/merge_requests/:merge_request_iid/rebase' do merge_request = find_project_merge_request(params[:merge_request_iid]) authorize_push_to_merge_request!(merge_request) - merge_request.rebase_async(current_user.id) + merge_request.rebase_async(current_user.id, skip_ci: params[:skip_ci]) status :accepted present rebase_in_progress: merge_request.rebase_in_progress? diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index c51417d2889..e40a5dde7ce 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -32,6 +32,8 @@ module API get do namespaces = current_user.admin ? Namespace.all : current_user.namespaces + namespaces = namespaces.include_gitlab_subscription if Gitlab.ee? + namespaces = namespaces.search(params[:search]) if params[:search].present? options = { with: Entities::Namespace, current_user: current_user } diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 89e4da5a42e..9575e8e9f36 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -24,6 +24,8 @@ module API desc: 'Return notes ordered by `created_at` or `updated_at` fields.' optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return notes sorted in `asc` or `desc` order.' + optional :activity_filter, type: String, values: UserPreference::NOTES_FILTERS.stringify_keys.keys, default: 'all_notes', + desc: 'The type of notables which are returned.' use :pagination end # rubocop: disable CodeReuse/ActiveRecord @@ -35,7 +37,8 @@ module API # at the DB query level (which we cannot in that case), the current # page can have less elements than :per_page even if # there's more than one page. - raw_notes = noteable.notes.with_metadata.reorder(order_options_with_tie_breaker) + notes_filter = UserPreference::NOTES_FILTERS[params[:activity_filter].to_sym] + raw_notes = noteable.notes.with_metadata.with_notes_filter(notes_filter).reorder(order_options_with_tie_breaker) # paginate() only works with a relation. This could lead to a # mismatch between the pagination headers info and the actual notes diff --git a/lib/api/pages.rb b/lib/api/pages.rb index 39c8f1e6bdf..ee7fe669519 100644 --- a/lib/api/pages.rb +++ b/lib/api/pages.rb @@ -17,9 +17,9 @@ module API delete ':id/pages' do authorize! :remove_pages, user_project - status 204 - ::Pages::DeleteService.new(user_project, current_user).execute + + no_content! end end end diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index 9f8c1e4f916..4c3d2d131ac 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -148,8 +148,9 @@ module API delete ":id/pages/domains/:domain", requirements: PAGES_DOMAINS_ENDPOINT_REQUIREMENTS do authorize! :update_pages, user_project - status 204 pages_domain.destroy + + no_content! end end end diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index aebf7d5fae1..8643854a655 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -69,7 +69,7 @@ module API milestone = user_project.milestones.find(params[:milestone_id]) Milestones::DestroyService.new(user_project, current_user).execute(milestone) - status(204) + no_content! end desc 'Get all issues for a single project milestone' do diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index b4545295d54..ecada843972 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -64,7 +64,8 @@ module API snippet_params = declared_params(include_missing: false).merge(request: request, api: true) snippet_params[:content] = snippet_params.delete(:code) if snippet_params[:code].present? - snippet = CreateSnippetService.new(user_project, current_user, snippet_params).execute + service_response = ::Snippets::CreateService.new(user_project, current_user, snippet_params).execute + snippet = service_response.payload[:snippet] render_spam_error! if snippet.spam? @@ -103,8 +104,8 @@ module API snippet_params[:content] = snippet_params.delete(:code) if snippet_params[:code].present? - UpdateSnippetService.new(user_project, current_user, snippet, - snippet_params).execute + service_response = ::Snippets::UpdateService.new(user_project, current_user, snippet_params).execute(snippet) + snippet = service_response.payload[:snippet] render_spam_error! if snippet.spam? @@ -127,7 +128,14 @@ module API authorize! :admin_project_snippet, snippet - destroy_conditionally!(snippet) + destroy_conditionally!(snippet) do |snippet| + service = ::Snippets::DestroyService.new(current_user, snippet) + response = service.execute + + if response.error? + render_api_error!({ error: response.message }, response.http_status) + end + end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/projects.rb b/lib/api/projects.rb index d1f99ea49ce..2271131ced3 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -90,18 +90,22 @@ module API def present_projects(projects, options = {}) projects = reorder_projects(projects) projects = apply_filters(projects) - projects = paginate(projects) - projects, options = with_custom_attributes(projects, options) - options = options.reverse_merge( - with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, - statistics: params[:statistics], - current_user: current_user, - license: false - ) - options[:with] = Entities::BasicProjectDetails if params[:simple] + records, options = paginate_with_strategies(projects) do |projects| + projects, options = with_custom_attributes(projects, options) + + options = options.reverse_merge( + with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, + statistics: params[:statistics], + current_user: current_user, + license: false + ) + options[:with] = Entities::BasicProjectDetails if params[:simple] + + [options[:with].prepare_relation(projects, options), options] + end - present options[:with].prepare_relation(projects, options), options + present records, options end def translate_params_for_compatibility(params) @@ -355,7 +359,7 @@ module API post ':id/unarchive' do authorize!(:archive_project, user_project) - ::Projects::UpdateService.new(@project, current_user, archived: false).execute + ::Projects::UpdateService.new(user_project, current_user, archived: false).execute present user_project, with: Entities::Project, current_user: current_user end @@ -443,7 +447,7 @@ module API ::Projects::UnlinkForkService.new(user_project, current_user).execute end - result ? status(204) : not_modified! + not_modified! unless result end desc 'Share the project with a group' do diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 2df6050967b..506d2b0f985 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -66,6 +66,8 @@ module API .execute if result[:status] == :success + log_release_created_audit_event(result[:release]) + present result[:release], with: Entities::Release, current_user: current_user else render_api_error!(result[:message], result[:http_status]) @@ -91,6 +93,9 @@ module API .execute if result[:status] == :success + log_release_updated_audit_event + log_release_milestones_updated_audit_event if result[:milestones_updated] + present result[:release], with: Entities::Release, current_user: current_user else render_api_error!(result[:message], result[:http_status]) @@ -147,6 +152,20 @@ module API def release @release ||= user_project.releases.find_by_tag(params[:tag]) end + + def log_release_created_audit_event(release) + # This is a separate method so that EE can extend its behaviour + end + + def log_release_updated_audit_event + # This is a separate method so that EE can extend its behaviour + end + + def log_release_milestones_updated_audit_event + # This is a separate method so that EE can extend its behaviour + end end end end + +API::Releases.prepend_if_ee('EE::API::Releases') diff --git a/lib/api/remote_mirrors.rb b/lib/api/remote_mirrors.rb index 8a085517ce9..95313966133 100644 --- a/lib/api/remote_mirrors.rb +++ b/lib/api/remote_mirrors.rb @@ -7,6 +7,8 @@ module API before do # TODO: Remove flag: https://gitlab.com/gitlab-org/gitlab/issues/38121 not_found! unless Feature.enabled?(:remote_mirrors_api, user_project) + + unauthorized! unless can?(current_user, :admin_remote_mirror, user_project) end params do @@ -20,11 +22,35 @@ module API use :pagination end get ':id/remote_mirrors' do - unauthorized! unless can?(current_user, :admin_remote_mirror, user_project) - present paginate(user_project.remote_mirrors), with: Entities::RemoteMirror end + + desc 'Update the attributes of a single remote mirror' do + success Entities::RemoteMirror + end + params do + requires :mirror_id, type: String, desc: 'The ID of a remote mirror' + optional :enabled, type: Boolean, desc: 'Determines if the mirror is enabled' + optional :only_protected_branches, type: Boolean, desc: 'Determines if only protected branches are mirrored' + end + put ':id/remote_mirrors/:mirror_id' do + mirror = user_project.remote_mirrors.find(params[:mirror_id]) + + mirror_params = declared_params(include_missing: false) + mirror_params[:id] = mirror_params.delete(:mirror_id) + update_params = { remote_mirrors_attributes: mirror_params } + + result = ::Projects::UpdateService + .new(user_project, current_user, update_params) + .execute + + if result[:status] == :success + present mirror.reset, with: Entities::RemoteMirror + else + render_api_error!(result[:message], result[:http_status]) + end + end end end end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index f383c541f8a..60cf9bf2c9c 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -200,6 +200,10 @@ module API status 202 header 'Job-Status', job.status header 'Range', "0-#{stream_size}" + + if Feature.enabled?(:runner_job_trace_update_interval_header, default_enabled: true) + header 'X-GitLab-Trace-Update-Interval', job.trace.update_interval.to_s + end end desc 'Authorize artifacts uploading for job' do diff --git a/lib/api/services.rb b/lib/api/services.rb index 03c51f65172..a3b5d2cc4b7 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -66,6 +66,15 @@ module API end end + desc 'Get all active project services' do + success Entities::ProjectServiceBasic + end + get ":id/services" do + services = user_project.services.active + + present services, with: Entities::ProjectServiceBasic + end + SERVICES.each do |service_slug, settings| desc "Set #{service_slug} service for project" params do diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index fd5422f2e2c..a7dab373b7f 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -75,7 +75,8 @@ module API end post do attrs = declared_params(include_missing: false).merge(request: request, api: true) - snippet = CreateSnippetService.new(nil, current_user, attrs).execute + service_response = ::Snippets::CreateService.new(nil, current_user, attrs).execute + snippet = service_response.payload[:snippet] render_spam_error! if snippet.spam? @@ -108,8 +109,8 @@ module API authorize! :update_personal_snippet, snippet attrs = declared_params(include_missing: false).merge(request: request, api: true) - - UpdateSnippetService.new(nil, current_user, snippet, attrs).execute + service_response = ::Snippets::UpdateService.new(nil, current_user, attrs).execute(snippet) + snippet = service_response.payload[:snippet] render_spam_error! if snippet.spam? @@ -133,7 +134,14 @@ module API authorize! :admin_personal_snippet, snippet - destroy_conditionally!(snippet) + destroy_conditionally!(snippet) do |snippet| + service = ::Snippets::DestroyService.new(current_user, snippet) + response = service.execute + + if response.error? + render_api_error!({ error: response.message }, response.http_status) + end + end end desc 'Get a raw snippet' do diff --git a/lib/api/users.rb b/lib/api/users.rb index b8c60f1969c..bf1fe4fc4a8 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -346,8 +346,9 @@ module API key = user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key - status 204 key.destroy + + no_content! end # rubocop: enable CodeReuse/ActiveRecord @@ -760,8 +761,9 @@ module API key = current_user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key - status 204 key.destroy + + no_content! end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/variables.rb b/lib/api/variables.rb index f022b9e665a..192b06b8a1b 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -111,9 +111,10 @@ module API variable = user_project.variables.find_by(key: params[:key]) not_found!('Variable') unless variable - # Variables don't have any timestamp. Therfore, destroy unconditionally. - status 204 + # Variables don't have a timestamp. Therefore, destroy unconditionally. variable.destroy + + no_content! end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index c5a5488950d..a2146406690 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -26,7 +26,7 @@ module API type: String, values: ProjectWiki::MARKUPS.values.map(&:to_s), default: 'markdown', - desc: 'Format of a wiki page. Available formats are markdown, rdoc, and asciidoc' + desc: 'Format of a wiki page. Available formats are markdown, rdoc, asciidoc and org' end end @@ -107,8 +107,9 @@ module API delete ':id/wikis/:slug' do authorize! :admin_wiki, user_project - status 204 WikiPages::DestroyService.new(user_project, current_user).execute(wiki_page) + + no_content! end desc 'Upload an attachment to the wiki repository' do diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index ca1f61055b0..5962403d488 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -43,15 +43,46 @@ module Banzai # Returns a String replaced with the return of the block. def self.references_in(text, pattern = object_class.reference_pattern) text.gsub(pattern) do |match| - symbol = $~[object_sym] - if object_class.reference_valid?(symbol) - yield match, symbol.to_i, $~[:project], $~[:namespace], $~ + if ident = identifier($~) + yield match, ident, $~[:project], $~[:namespace], $~ else match end end end + def self.identifier(match_data) + symbol = symbol_from_match(match_data) + + parse_symbol(symbol, match_data) if object_class.reference_valid?(symbol) + end + + def identifier(match_data) + self.class.identifier(match_data) + end + + def self.symbol_from_match(match) + key = object_sym + match[key] if match.names.include?(key.to_s) + end + + # Transform a symbol extracted from the text to a meaningful value + # In most cases these will be integers, so we call #to_i by default + # + # This method has the contract that if a string `ref` refers to a + # record `record`, then `parse_symbol(ref) == record_identifier(record)`. + def self.parse_symbol(symbol, match_data) + symbol.to_i + end + + # We assume that most classes are identifying records by ID. + # + # This method has the contract that if a string `ref` refers to a + # record `record`, then `class.parse_symbol(ref) == record_identifier(record)`. + def record_identifier(record) + record.id + end + def object_class self.class.object_class end @@ -265,8 +296,10 @@ module Banzai @references_per[parent_type] ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } - - regex = Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern) + regex = [ + object_class.link_reference_pattern, + object_class.reference_pattern + ].compact.reduce { |a, b| Regexp.union(a, b) } nodes.each do |node| node.to_html.scan(regex) do @@ -276,8 +309,9 @@ module Banzai full_group_path($~[:group]) end - symbol = $~[object_sym] - refs[path] << symbol if object_class.reference_valid?(symbol) + if ident = identifier($~) + refs[path] << ident + end end end diff --git a/lib/banzai/filter/base_relative_link_filter.rb b/lib/banzai/filter/base_relative_link_filter.rb new file mode 100644 index 00000000000..eca105ce9d9 --- /dev/null +++ b/lib/banzai/filter/base_relative_link_filter.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'uri' + +module Banzai + module Filter + class BaseRelativeLinkFilter < HTML::Pipeline::Filter + include Gitlab::Utils::StrongMemoize + + protected + + def linkable_attributes + strong_memoize(:linkable_attributes) do + attrs = [] + + attrs += doc.search('a:not(.gfm)').map do |el| + el.attribute('href') + end + + attrs += doc.search('img:not(.gfm), video:not(.gfm), audio:not(.gfm)').flat_map do |el| + [el.attribute('src'), el.attribute('data-src')] + end + + attrs.reject do |attr| + attr.blank? || attr.value.start_with?('//') + end + end + end + + def relative_url_root + Gitlab.config.gitlab.relative_url_root.presence || '/' + end + + def project + context[:project] + end + + private + + def unescape_and_scrub_uri(uri) + Addressable::URI.unescape(uri).scrub + end + end + end +end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index e1d7b36b9a2..3df003a88fa 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -37,6 +37,11 @@ module Banzai end end + # The default behaviour is `#to_i` - we just pass the hash through. + def self.parse_symbol(sha_hash, _match) + sha_hash + end + def url_for_object(commit, project) h = Gitlab::Routing.url_helpers @@ -65,10 +70,6 @@ module Banzai private - def record_identifier(record) - record.id - end - def parent_records(parent, ids) parent.commits_by(oids: ids.to_a) end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index caba8955bac..1a75cd14b11 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require "nokogiri" -require "asciidoctor-plantuml/plantuml" +require "asciidoctor_plantuml/plantuml" module Banzai module Filter diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/repository_link_filter.rb index 4f257189f8e..14cd607cc50 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/repository_link_filter.rb @@ -4,19 +4,17 @@ require 'uri' module Banzai module Filter - # HTML filter that "fixes" relative links to uploads or files in a repository. + # HTML filter that "fixes" relative links to files in a repository. # # Context options: # :commit - # :group # :current_user # :project # :project_wiki # :ref # :requested_path - class RelativeLinkFilter < HTML::Pipeline::Filter - include Gitlab::Utils::StrongMemoize - + # :system_note + class RepositoryLinkFilter < BaseRelativeLinkFilter def call return doc if context[:system_note] @@ -26,7 +24,9 @@ module Banzai load_uri_types linkable_attributes.each do |attr| - process_link_attr(attr) + if linkable_files? && repo_visible_to_user? + process_link_to_repository_attr(attr) + end end doc @@ -35,8 +35,8 @@ module Banzai protected def load_uri_types - return unless linkable_files? return unless linkable_attributes.present? + return unless linkable_files? return {} unless repository @uri_types = request_path.present? ? get_uri_types([request_path]) : {} @@ -57,24 +57,6 @@ module Banzai end end - def linkable_attributes - strong_memoize(:linkable_attributes) do - attrs = [] - - attrs += doc.search('a:not(.gfm)').map do |el| - el.attribute('href') - end - - attrs += doc.search('img, video, audio').flat_map do |el| - [el.attribute('src'), el.attribute('data-src')] - end - - attrs.reject do |attr| - attr.blank? || attr.value.start_with?('//') - end - end - end - def get_uri_types(paths) return {} if paths.empty? @@ -107,39 +89,6 @@ module Banzai rescue URI::Error, Addressable::URI::InvalidURIError end - def process_link_attr(html_attr) - if html_attr.value.start_with?('/uploads/') - process_link_to_upload_attr(html_attr) - elsif linkable_files? && repo_visible_to_user? - process_link_to_repository_attr(html_attr) - end - end - - def process_link_to_upload_attr(html_attr) - path_parts = [unescape_and_scrub_uri(html_attr.value)] - - if project - path_parts.unshift(relative_url_root, project.full_path) - elsif group - path_parts.unshift(relative_url_root, 'groups', group.full_path, '-') - else - path_parts.unshift(relative_url_root) - end - - begin - path = Addressable::URI.escape(File.join(*path_parts)) - rescue Addressable::URI::InvalidURIError - return - end - - html_attr.value = - if context[:only_path] - path - else - Addressable::URI.join(Gitlab.config.gitlab.base_url, path).to_s - end - end - def process_link_to_repository_attr(html_attr) uri = URI(html_attr.value) @@ -239,10 +188,6 @@ module Banzai @current_commit ||= context[:commit] || repository.commit(ref) end - def relative_url_root - Gitlab.config.gitlab.relative_url_root.presence || '/' - end - def repo_visible_to_user? project && Ability.allowed?(current_user, :download_code, project) end @@ -251,14 +196,6 @@ module Banzai context[:ref] || project.default_branch end - def group - context[:group] - end - - def project - context[:project] - end - def current_user context[:current_user] end @@ -266,12 +203,6 @@ module Banzai def repository @repository ||= project&.repository end - - private - - def unescape_and_scrub_uri(uri) - Addressable::URI.unescape(uri).scrub - end end end end diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb new file mode 100644 index 00000000000..023c1288367 --- /dev/null +++ b/lib/banzai/filter/upload_link_filter.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'uri' + +module Banzai + module Filter + # HTML filter that "fixes" links to uploads. + # + # Context options: + # :group + # :only_path + # :project + # :system_note + class UploadLinkFilter < BaseRelativeLinkFilter + def call + return doc if context[:system_note] + + linkable_attributes.each do |attr| + process_link_to_upload_attr(attr) + end + + doc + end + + protected + + def process_link_to_upload_attr(html_attr) + return unless html_attr.value.start_with?('/uploads/') + + path_parts = [unescape_and_scrub_uri(html_attr.value)] + + if project + path_parts.unshift(relative_url_root, project.full_path) + elsif group + path_parts.unshift(relative_url_root, 'groups', group.full_path, '-') + else + path_parts.unshift(relative_url_root) + end + + begin + path = Addressable::URI.escape(File.join(*path_parts)) + rescue Addressable::URI::InvalidURIError + return + end + + html_attr.value = + if context[:only_path] + path + else + Addressable::URI.join(Gitlab.config.gitlab.base_url, path).to_s + end + + html_attr.parent.add_class('gfm') + end + + def group + context[:group] + end + end + end +end diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index fe629a23ff1..5e02d972614 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -16,7 +16,10 @@ module Banzai [ Filter::ReferenceRedactorFilter, Filter::InlineMetricsRedactorFilter, - Filter::RelativeLinkFilter, + # UploadLinkFilter must come before RepositoryLinkFilter to + # prevent unnecessary Gitaly calls from being made. + Filter::UploadLinkFilter, + Filter::RepositoryLinkFilter, Filter::IssuableStateFilter, Filter::SuggestionFilter ] diff --git a/lib/banzai/pipeline/relative_link_pipeline.rb b/lib/banzai/pipeline/relative_link_pipeline.rb deleted file mode 100644 index 88651892acc..00000000000 --- a/lib/banzai/pipeline/relative_link_pipeline.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Pipeline - class RelativeLinkPipeline < BasePipeline - def self.filters - FilterArray[ - Filter::RelativeLinkFilter - ] - end - end - end -end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 8419769085a..9160c0e14cf 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -177,7 +177,7 @@ module Banzai collection.where(id: to_query).each { |row| cache[row.id] = row } end - cache.values_at(*ids).compact + ids.uniq.map { |id| cache[id] }.compact else collection.where(id: ids) end diff --git a/lib/feature.rb b/lib/feature.rb index 88b0d871c3a..543512b1598 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -52,6 +52,10 @@ class Feature # use `default_enabled: true` to default the flag to being `enabled` # unless set explicitly. The default is `disabled` def enabled?(key, thing = nil, default_enabled: false) + # During setup the database does not exist yet. So we haven't stored a value + # for the feature yet and return the default. + return default_enabled unless Gitlab::Database.exists? + feature = Feature.get(key) # If we're not default enabling the flag or the feature has been set, always evaluate. diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index 625db1fce32..2bd55c36a03 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -7,6 +7,7 @@ class Feature # Server feature flags should use '_' to separate words. SERVER_FEATURE_FLAGS = %w[ + cache_invalidator inforef_uploadpack_cache get_tag_messages_go filter_shas_with_signatures_go diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 0e6db54eb46..f2bff51df38 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -100,8 +100,8 @@ module Gitlab end def self.process_name - return 'sidekiq' if Sidekiq.server? - return 'console' if defined?(Rails::Console) + return 'sidekiq' if Gitlab::Runtime.sidekiq? + return 'console' if Gitlab::Runtime.console? return 'test' if Rails.env.test? 'web' diff --git a/lib/gitlab/app_json_logger.rb b/lib/gitlab/app_json_logger.rb new file mode 100644 index 00000000000..e29b205e1bf --- /dev/null +++ b/lib/gitlab/app_json_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class AppJsonLogger < Gitlab::JsonLogger + def self.file_name_noext + 'application_json' + end + end +end diff --git a/lib/gitlab/app_logger.rb b/lib/gitlab/app_logger.rb index 5edec8b3efe..3f5e9adf925 100644 --- a/lib/gitlab/app_logger.rb +++ b/lib/gitlab/app_logger.rb @@ -1,13 +1,15 @@ # frozen_string_literal: true module Gitlab - class AppLogger < Gitlab::Logger - def self.file_name_noext - 'application' + class AppLogger < Gitlab::MultiDestinationLogger + LOGGERS = [Gitlab::AppTextLogger, Gitlab::AppJsonLogger].freeze + + def self.loggers + LOGGERS end - def format_message(severity, timestamp, progname, msg) - "#{timestamp.to_s(:long)}: #{msg}\n" + def self.primary_logger + Gitlab::AppTextLogger end end end diff --git a/lib/gitlab/app_text_logger.rb b/lib/gitlab/app_text_logger.rb new file mode 100644 index 00000000000..5b0439f43ad --- /dev/null +++ b/lib/gitlab/app_text_logger.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + class AppTextLogger < Gitlab::Logger + def self.file_name_noext + 'application' + end + + def format_message(severity, timestamp, progname, msg) + "#{timestamp.utc.iso8601(3)}: #{msg}\n" + end + end +end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb new file mode 100644 index 00000000000..71dbfea70e8 --- /dev/null +++ b/lib/gitlab/application_context.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + # A GitLab-rails specific accessor for `Labkit::Logging::ApplicationContext` + class ApplicationContext + include Gitlab::Utils::LazyAttributes + + Attribute = Struct.new(:name, :type) + + APPLICATION_ATTRIBUTES = [ + Attribute.new(:project, Project), + Attribute.new(:namespace, Namespace), + Attribute.new(:user, User) + ].freeze + + def self.with_context(args, &block) + application_context = new(**args) + Labkit::Context.with_context(application_context.to_lazy_hash, &block) + end + + def self.push(args) + application_context = new(**args) + Labkit::Context.push(application_context.to_lazy_hash) + end + + def initialize(**args) + unknown_attributes = args.keys - APPLICATION_ATTRIBUTES.map(&:name) + raise ArgumentError, "#{unknown_attributes} are not known keys" if unknown_attributes.any? + + @set_values = args.keys + + assign_attributes(args) + end + + def to_lazy_hash + {}.tap do |hash| + hash[:user] = -> { username } if set_values.include?(:user) + hash[:project] = -> { project_path } if set_values.include?(:project) + hash[:root_namespace] = -> { root_namespace_path } if include_namespace? + end + end + + private + + attr_reader :set_values + + APPLICATION_ATTRIBUTES.each do |attr| + lazy_attr_reader attr.name, type: attr.type + end + + def assign_attributes(values) + values.slice(*APPLICATION_ATTRIBUTES.map(&:name)).each do |name, value| + instance_variable_set("@#{name}", value) + end + end + + def project_path + project&.full_path + end + + def username + user&.username + end + + def root_namespace_path + if namespace + namespace.full_path_components.first + else + project&.full_path_components&.first + end + end + + def include_namespace? + set_values.include?(:namespace) || set_values.include?(:project) + end + end +end diff --git a/lib/gitlab/asciidoc/include_processor.rb b/lib/gitlab/asciidoc/include_processor.rb index c6fbf540e9c..6e0b7ce60ba 100644 --- a/lib/gitlab/asciidoc/include_processor.rb +++ b/lib/gitlab/asciidoc/include_processor.rb @@ -13,7 +13,7 @@ module Gitlab super(logger: Gitlab::AppLogger) @context = context - @repository = context[:project].try(:repository) + @repository = context[:repository] || context[:project].try(:repository) # Note: Asciidoctor calls #freeze on extensions, so we can't set new # instance variables after initialization. @@ -111,7 +111,7 @@ module Gitlab end def ref - context[:ref] || context[:project].default_branch + context[:ref] || repository&.root_ref end def requested_path diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index dfdba617cb6..821c68dbedc 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -54,7 +54,7 @@ module Gitlab Gitlab::Auth::Result.new rate_limit!(rate_limiter, success: result.success?, login: login) - Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) + look_to_limit_user(result.actor) return result if result.success? || authenticate_using_internal_or_ldap_password? @@ -129,6 +129,10 @@ module Gitlab ::Ci::Build::CI_REGISTRY_USER == login end + def look_to_limit_user(actor) + Gitlab::Auth::UniqueIpsLimiter.limit_user!(actor) if actor.is_a?(User) + end + def authenticate_using_internal_or_ldap_password? Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::Auth::LDAP::Config.enabled? end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 33cbb070c2f..fe61d9fe8ca 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -25,9 +25,10 @@ module Gitlab PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN' PRIVATE_TOKEN_PARAM = :private_token - JOB_TOKEN_HEADER = "HTTP_JOB_TOKEN".freeze + JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze JOB_TOKEN_PARAM = :job_token RUNNER_TOKEN_PARAM = :token + RUNNER_JOB_TOKEN_PARAM = :token # Check the Rails session for valid authentication details def find_user_from_warden @@ -57,11 +58,13 @@ module Gitlab def find_user_from_job_token return unless route_authentication_setting[:job_token_allowed] - token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s - return unless token.present? + token = current_request.params[JOB_TOKEN_PARAM].presence || + current_request.params[RUNNER_JOB_TOKEN_PARAM].presence || + current_request.env[JOB_TOKEN_HEADER].presence + return unless token job = ::Ci::Build.find_by_token(token) - raise ::Gitlab::Auth::UnauthorizedError unless job + raise UnauthorizedError unless job @current_authenticated_job = job # rubocop:disable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index 34ccff588f4..c6216fa9cad 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -33,7 +33,8 @@ module Gitlab find_user_from_web_access_token(request_format) || find_user_from_feed_token(request_format) || find_user_from_static_object_token(request_format) || - find_user_from_basic_auth_job + find_user_from_basic_auth_job || + find_user_from_job_token rescue Gitlab::Auth::AuthenticationError nil end @@ -45,6 +46,14 @@ module Gitlab rescue Gitlab::Auth::AuthenticationError false end + + private + + def route_authentication_setting + @route_authentication_setting ||= { + job_token_allowed: api_request? + } + end end end end diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index 97e78ecf094..74f7fdfc180 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -8,7 +8,7 @@ module Gitlab class << self def limit_user_id!(user_id) if config.unique_ips_limit_enabled - ip = RequestContext.client_ip + ip = RequestContext.instance.client_ip unique_ips = update_and_return_ips_count(user_id, ip) raise TooManyIps.new(user_id, ip, unique_ips) if unique_ips > config.unique_ips_limit_per_user diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index 61e0a075018..ddd6b11eebb 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -78,6 +78,20 @@ module Gitlab end def self.migration_class_for(class_name) + # We don't pass class name with Gitlab::BackgroundMigration:: prefix anymore + # but some jobs could be already spawned so we need to have some backward compatibility period. + # Can be removed since 13.x + full_class_name_prefix_regexp = /\A(::)?Gitlab::BackgroundMigration::/ + + if class_name.match(full_class_name_prefix_regexp) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + StandardError.new("Full class name is used"), + class_name: class_name + ) + + class_name = class_name.sub(full_class_name_prefix_regexp, '') + end + const_get(class_name, false) end diff --git a/lib/gitlab/background_migration/activate_prometheus_services_for_shared_cluster_applications.rb b/lib/gitlab/background_migration/activate_prometheus_services_for_shared_cluster_applications.rb new file mode 100644 index 00000000000..19f5821d449 --- /dev/null +++ b/lib/gitlab/background_migration/activate_prometheus_services_for_shared_cluster_applications.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Create missing PrometheusServices records or sets active attribute to true + # for all projects which belongs to cluster with Prometheus Application installed. + class ActivatePrometheusServicesForSharedClusterApplications + module Migratable + # Migration model namespace isolated from application code. + class PrometheusService < ActiveRecord::Base + self.inheritance_column = :_type_disabled + self.table_name = 'services' + + default_scope { where("services.type = 'PrometheusService'") } + + def self.for_project(project_id) + new( + project_id: project_id, + active: true, + properties: '{}', + type: 'PrometheusService', + template: false, + push_events: true, + issues_events: true, + merge_requests_events: true, + tag_push_events: true, + note_events: true, + category: 'monitoring', + default: false, + wiki_page_events: true, + pipeline_events: true, + confidential_issues_events: true, + commit_events: true, + job_events: true, + confidential_note_events: true, + deployment_events: false + ) + end + + def managed? + properties == '{}' + end + end + end + + def perform(project_id) + service = Migratable::PrometheusService.find_by(project_id: project_id) || Migratable::PrometheusService.for_project(project_id) + service.update!(active: true) if service.managed? + end + end + end +end diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb index 3c26982729d..79f38aed9f1 100644 --- a/lib/gitlab/background_migration/archive_legacy_traces.rb +++ b/lib/gitlab/background_migration/archive_legacy_traces.rb @@ -11,7 +11,6 @@ module Gitlab # So we chose a way to use ::Ci::Build directly and we don't change the `archive!` method until 11.1 ::Ci::Build.finished.without_archived_trace .where(id: start_id..stop_id).find_each do |build| - build.trace.archive! rescue => e Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger diff --git a/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb b/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb new file mode 100644 index 00000000000..83d60d2db19 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class BackfillVersionDataFromGitaly + def perform(issue_id) + end + end + end +end + +Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly.prepend_if_ee('EE::Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly') diff --git a/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb b/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb new file mode 100644 index 00000000000..85bcf8558f2 --- /dev/null +++ b/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class GenerateGitlabSubscriptions + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::GenerateGitlabSubscriptions.prepend_if_ee('EE::Gitlab::BackgroundMigration::GenerateGitlabSubscriptions') diff --git a/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb b/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb new file mode 100644 index 00000000000..27b984b4531 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class MigrateApproverToApprovalRules + # @param target_type [String] class of target, either 'MergeRequest' or 'Project' + # @param target_id [Integer] id of target + def perform(target_type, target_id, sync_code_owner_rule: true) + end + end + end +end + +Gitlab::BackgroundMigration::MigrateApproverToApprovalRules.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRules') diff --git a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb new file mode 100644 index 00000000000..053b7363286 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class MigrateApproverToApprovalRulesCheckProgress + def perform + end + end + end +end + +Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress') diff --git a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb new file mode 100644 index 00000000000..130f97b09d7 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class MigrateApproverToApprovalRulesInBatch + def perform(start_id, end_id) + end + end + end +end + +Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch') diff --git a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb new file mode 100644 index 00000000000..899f381e911 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class is responsible to update all sha256 fingerprints within the keys table + class MigrateFingerprintSha256WithinKeys + # Temporary AR table for keys + class Key < ActiveRecord::Base + include EachBatch + + self.table_name = 'keys' + self.inheritance_column = :_type_disabled + end + + TEMP_TABLE = 'tmp_fingerprint_sha256_migration' + + def perform(start_id, stop_id) + ActiveRecord::Base.transaction do + execute(<<~SQL) + CREATE TEMPORARY TABLE #{TEMP_TABLE} + (id bigint primary key, fingerprint_sha256 bytea not null) + ON COMMIT DROP + SQL + + fingerprints = [] + Key.where(id: start_id..stop_id, fingerprint_sha256: nil).find_each do |regular_key| + if fingerprint = generate_ssh_public_key(regular_key.key) + bytea = ActiveRecord::Base.connection.escape_bytea(Base64.decode64(fingerprint)) + + fingerprints << { + id: regular_key.id, + fingerprint_sha256: bytea + } + end + end + + Gitlab::Database.bulk_insert(TEMP_TABLE, fingerprints) + + execute("ANALYZE #{TEMP_TABLE}") + + execute(<<~SQL) + UPDATE keys + SET fingerprint_sha256 = t.fingerprint_sha256 + FROM #{TEMP_TABLE} t + WHERE keys.id = t.id + SQL + end + end + + private + + def generate_ssh_public_key(regular_key) + Gitlab::SSHPublicKey.new(regular_key).fingerprint("SHA256")&.gsub("SHA256:", "") + end + + def execute(query) + ActiveRecord::Base.connection.execute(query) + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb new file mode 100644 index 00000000000..14e14f28439 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration takes all issue trackers + # and move data from properties to data field tables (jira_tracker_data and issue_tracker_data) + class MigrateIssueTrackersSensitiveData + delegate :select_all, :execute, :quote_string, to: :connection + + # we need to define this class and set fields encryption + class IssueTrackerData < ApplicationRecord + self.table_name = 'issue_tracker_data' + + def self.encryption_options + { + key: Settings.attr_encrypted_db_key_base_32, + encode: true, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm' + } + end + + attr_encrypted :project_url, encryption_options + attr_encrypted :issues_url, encryption_options + attr_encrypted :new_issue_url, encryption_options + end + + # we need to define this class and set fields encryption + class JiraTrackerData < ApplicationRecord + self.table_name = 'jira_tracker_data' + + def self.encryption_options + { + key: Settings.attr_encrypted_db_key_base_32, + encode: true, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm' + } + end + + attr_encrypted :url, encryption_options + attr_encrypted :api_url, encryption_options + attr_encrypted :username, encryption_options + attr_encrypted :password, encryption_options + end + + def perform(start_id, stop_id) + columns = 'id, properties, title, description, type' + batch_condition = "id >= #{start_id} AND id <= #{stop_id} AND category = 'issue_tracker' \ + AND properties IS NOT NULL AND properties != '{}' AND properties != ''" + + data_subselect = "SELECT 1 \ + FROM jira_tracker_data \ + WHERE jira_tracker_data.service_id = services.id \ + UNION SELECT 1 \ + FROM issue_tracker_data \ + WHERE issue_tracker_data.service_id = services.id" + + query = "SELECT #{columns} FROM services WHERE #{batch_condition} AND NOT EXISTS (#{data_subselect})" + + migrated_ids = [] + data_to_insert(query).each do |table, data| + service_ids = data.map { |s| s['service_id'] } + + next if service_ids.empty? + + migrated_ids += service_ids + Gitlab::Database.bulk_insert(table, data) + end + + return if migrated_ids.empty? + + move_title_description(migrated_ids) + end + + private + + def data_to_insert(query) + data = { 'jira_tracker_data' => [], 'issue_tracker_data' => [] } + select_all(query).each do |service| + begin + properties = JSON.parse(service['properties']) + rescue JSON::ParserError + logger.warn( + message: 'Properties data not parsed - invalid json', + service_id: service['id'], + properties: service['properties'] + ) + next + end + + if service['type'] == 'JiraService' + row = data_row(JiraTrackerData, jira_mapping(properties), service) + key = 'jira_tracker_data' + else + row = data_row(IssueTrackerData, issue_tracker_mapping(properties), service) + key = 'issue_tracker_data' + end + + data[key] << row if row + end + + data + end + + def data_row(klass, mapping, service) + base_params = { service_id: service['id'], created_at: Time.current, updated_at: Time.current } + klass.new(mapping).slice(*klass.column_names).compact.merge(base_params) + end + + def move_title_description(service_ids) + query = "UPDATE services SET \ + title = cast(properties as json)->>'title', \ + description = cast(properties as json)->>'description' \ + WHERE id IN (#{service_ids.join(',')}) AND title IS NULL AND description IS NULL" + + execute(query) + end + + def jira_mapping(properties) + { + url: properties['url'], + api_url: properties['api_url'], + username: properties['username'], + password: properties['password'] + } + end + + def issue_tracker_mapping(properties) + { + project_url: properties['project_url'], + issues_url: properties['issues_url'], + new_issue_url: properties['new_issue_url'] + } + end + + def connection + @connection ||= ActiveRecord::Base.connection + end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + end + end +end diff --git a/lib/gitlab/background_migration/move_epic_issues_after_epics.rb b/lib/gitlab/background_migration/move_epic_issues_after_epics.rb new file mode 100644 index 00000000000..dc982e703d1 --- /dev/null +++ b/lib/gitlab/background_migration/move_epic_issues_after_epics.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class MoveEpicIssuesAfterEpics + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics.prepend_if_ee('EE::Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics') diff --git a/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb b/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb new file mode 100644 index 00000000000..c3c0db2495c --- /dev/null +++ b/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This background migration creates any approver rule records according + # to the given merge request IDs range. A _single_ INSERT is issued for the given range. + class PopulateAnyApprovalRuleForMergeRequests + def perform(from_id, to_id) + end + end + end +end + +Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests') diff --git a/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb b/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb new file mode 100644 index 00000000000..2243c7531c0 --- /dev/null +++ b/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This background migration creates any approver rule records according + # to the given project IDs range. A _single_ INSERT is issued for the given range. + class PopulateAnyApprovalRuleForProjects + def perform(from_id, to_id) + end + end + end +end + +Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects') diff --git a/lib/gitlab/background_migration/prune_orphaned_geo_events.rb b/lib/gitlab/background_migration/prune_orphaned_geo_events.rb new file mode 100644 index 00000000000..8b16db8be35 --- /dev/null +++ b/lib/gitlab/background_migration/prune_orphaned_geo_events.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true +# +# rubocop:disable Style/Documentation + +# This job is added to fix https://gitlab.com/gitlab-org/gitlab/issues/30229 +# It's not used anywhere else. +# Can be removed in GitLab 13.* +module Gitlab + module BackgroundMigration + class PruneOrphanedGeoEvents + def perform(table_name) + end + end + end +end + +Gitlab::BackgroundMigration::PruneOrphanedGeoEvents.prepend_if_ee('EE::Gitlab::BackgroundMigration::PruneOrphanedGeoEvents') diff --git a/lib/gitlab/background_migration/update_authorized_keys_file_since.rb b/lib/gitlab/background_migration/update_authorized_keys_file_since.rb new file mode 100644 index 00000000000..dd80d4bab1a --- /dev/null +++ b/lib/gitlab/background_migration/update_authorized_keys_file_since.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class UpdateAuthorizedKeysFileSince + def perform(cutoff_datetime) + end + end + end +end + +Gitlab::BackgroundMigration::UpdateAuthorizedKeysFileSince.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateAuthorizedKeysFileSince') diff --git a/lib/gitlab/background_migration/update_vulnerability_confidence.rb b/lib/gitlab/background_migration/update_vulnerability_confidence.rb new file mode 100644 index 00000000000..6ffaa836f3c --- /dev/null +++ b/lib/gitlab/background_migration/update_vulnerability_confidence.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class UpdateVulnerabilityConfidence + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence') diff --git a/lib/gitlab/backtrace_cleaner.rb b/lib/gitlab/backtrace_cleaner.rb new file mode 100644 index 00000000000..30ec99808f7 --- /dev/null +++ b/lib/gitlab/backtrace_cleaner.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module BacktraceCleaner + IGNORE_BACKTRACES = %w[ + config/initializers + ee/lib/gitlab/middleware/ + lib/gitlab/correlation_id.rb + lib/gitlab/database/load_balancing/ + lib/gitlab/etag_caching/ + lib/gitlab/i18n.rb + lib/gitlab/metrics/ + lib/gitlab/middleware/ + lib/gitlab/performance_bar/ + lib/gitlab/profiler.rb + lib/gitlab/query_limiting/ + lib/gitlab/request_context.rb + lib/gitlab/request_profiler/ + lib/gitlab/sidekiq_logging/ + lib/gitlab/sidekiq_middleware/ + lib/gitlab/sidekiq_status/ + lib/gitlab/tracing/ + lib/gitlab/webpack/dev_server_middleware.rb + ].freeze + + IGNORED_BACKTRACES_REGEXP = Regexp.union(IGNORE_BACKTRACES).freeze + + def self.clean_backtrace(backtrace) + return unless backtrace + + Array(Rails.backtrace_cleaner.clean(backtrace)).reject do |line| + line.match(IGNORED_BACKTRACES_REGEXP) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 67118aed549..3a087a3ef83 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -42,7 +42,7 @@ module Gitlab end def store_pull_request_error(pull_request, ex) - backtrace = Gitlab::Profiler.clean_backtrace(ex.backtrace) + backtrace = Gitlab::BacktraceCleaner.clean_backtrace(ex.backtrace) error = { type: :pull_request, iid: pull_request.iid, errors: ex.message, trace: backtrace, raw_response: pull_request.raw } Gitlab::ErrorTracking.log_exception(ex, error) @@ -182,7 +182,6 @@ module Gitlab target_branch_sha: target_branch_sha, state: pull_request.state, author_id: gitlab_user_id(project, pull_request.author), - assignee_id: nil, created_at: pull_request.created_at, updated_at: pull_request.updated_at ) diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index b7b2fe115c1..886fbaaff48 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -211,7 +211,6 @@ module Gitlab target_branch_sha: pull_request.target_branch_sha, state_id: MergeRequest.available_states[pull_request.state], author_id: author_id, - assignee_id: nil, created_at: pull_request.created_at, updated_at: pull_request.updated_at } diff --git a/lib/gitlab/ci/config/entry/includes.rb b/lib/gitlab/ci/config/entry/includes.rb index 43e74dfd628..24d0e27e3a7 100644 --- a/lib/gitlab/ci/config/entry/includes.rb +++ b/lib/gitlab/ci/config/entry/includes.rb @@ -12,6 +12,15 @@ module Gitlab validations do validates :config, array_or_string: true + + validate do + next unless opt(:max_size) + next unless config.is_a?(Array) + + if config.size > opt(:max_size) + errors.add(:config, "is too long (maximum is #{opt(:max_size)})") + end + end end def self.aspects diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 6a55b8cda57..124581c961f 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -16,7 +16,8 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except rules type image services allow_failure type stage when start_in artifacts cache dependencies before_script needs after_script variables - environment coverage retry parallel extends interruptible timeout].freeze + environment coverage retry parallel extends interruptible timeout + resource_group release].freeze REQUIRED_BY_NEEDS = %i[stage].freeze @@ -34,6 +35,12 @@ module Gitlab message: 'key may not be used with `rules`' }, if: :has_rules? + validates :config, + disallowed_keys: { + in: %i[release], + message: 'release features are not enabled' + }, + unless: -> { Feature.enabled?(:ci_release_generation, default_enabled: false) } with_options allow_nil: true do validates :allow_failure, boolean: true @@ -48,16 +55,18 @@ module Gitlab validates :dependencies, array_of_strings: true validates :extends, array_of_strings_or_string: true validates :rules, array_of_hashes: true + validates :resource_group, type: String end validates :start_in, duration: { limit: '1 week' }, if: :delayed? validates :start_in, absence: true, if: -> { has_rules? || !delayed? } - validate do + validate on: :composed do next unless dependencies.present? - next unless needs.present? + next unless needs_value.present? + + missing_needs = dependencies - needs_value[:job].pluck(:name) # rubocop:disable CodeReuse/ActiveRecord (Array#pluck) - missing_needs = dependencies - needs if missing_needs.any? errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") end @@ -149,14 +158,18 @@ module Gitlab description: 'Coverage configuration for this job.', inherit: false + entry :release, Entry::Release, + description: 'This job will produce a release.', + inherit: false + helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, :artifacts, :environment, :coverage, :retry, :rules, - :parallel, :needs, :interruptible + :parallel, :needs, :interruptible, :release attributes :script, :tags, :allow_failure, :when, :dependencies, :needs, :retry, :parallel, :extends, :start_in, :rules, - :interruptible, :timeout + :interruptible, :timeout, :resource_group, :release def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -241,9 +254,11 @@ module Gitlab interruptible: interruptible_defined? ? interruptible_value : nil, timeout: has_timeout? ? ChronicDuration.parse(timeout.to_s) : nil, artifacts: artifacts_value, + release: release_value, after_script: after_script_value, ignore: ignored?, - needs: needs_defined? ? needs_value : nil } + needs: needs_defined? ? needs_value : nil, + resource_group: resource_group } end end end diff --git a/lib/gitlab/ci/config/entry/release.rb b/lib/gitlab/ci/config/entry/release.rb new file mode 100644 index 00000000000..3eceaa0ccd9 --- /dev/null +++ b/lib/gitlab/ci/config/entry/release.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a release configuration. + # + class Release < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[tag_name name description assets].freeze + attributes %i[tag_name name assets].freeze + + # Attributable description conflicts with + # ::Gitlab::Config::Entry::Node.description + def has_description? + true + end + + def description + config[:description] + end + + entry :assets, Entry::Release::Assets, description: 'Release assets.' + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + validates :tag_name, presence: true + validates :description, type: String, presence: true + end + + helpers :assets + + def value + @config[:assets] = assets_value if @config.key?(:assets) + @config + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/release/assets.rb b/lib/gitlab/ci/config/entry/release/assets.rb new file mode 100644 index 00000000000..82ed39f51e0 --- /dev/null +++ b/lib/gitlab/ci/config/entry/release/assets.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a configuration of release assets. + # + class Release + class Assets < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[links].freeze + attributes ALLOWED_KEYS + + entry :links, Entry::Release::Assets::Links, description: 'Release assets:links.' + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + validates :links, array_of_hashes: true, presence: true + end + + helpers :links + + def value + @config[:links] = links_value if @config.key?(:links) + @config + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/release/assets/link.rb b/lib/gitlab/ci/config/entry/release/assets/link.rb new file mode 100644 index 00000000000..8e8fcde16a3 --- /dev/null +++ b/lib/gitlab/ci/config/entry/release/assets/link.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a configuration of release:assets:links. + # + class Release + class Assets + class Link < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[name url].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + + validates :name, type: String, presence: true + validates :url, presence: true, addressable_url: true + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/release/assets/links.rb b/lib/gitlab/ci/config/entry/release/assets/links.rb new file mode 100644 index 00000000000..b791d173d54 --- /dev/null +++ b/lib/gitlab/ci/config/entry/release/assets/links.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a configuration of release:assets:links. + # + class Release + class Assets + class Links < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + + entry :link, Entry::Release::Assets::Link, description: 'Release assets:links:link.' + + validations do + validates :config, type: Array, presence: true + end + + def skip_config_hash_validation? + true + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index eeaac52eefd..f984d7d397a 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -11,7 +11,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management metrics].freeze + ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management license_scanning metrics].freeze attributes ALLOWED_KEYS @@ -28,6 +28,7 @@ module Gitlab validates :dast, array_of_strings_or_string: true validates :performance, array_of_strings_or_string: true validates :license_management, array_of_strings_or_string: true + validates :license_scanning, array_of_strings_or_string: true validates :metrics, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index c2df419cca0..6a16e6df23d 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -10,7 +10,7 @@ module Gitlab :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update, + :chat_data, :allow_mirror_update, :bridge, # These attributes are set by Chains during processing: :config_content, :config_processor, :stage_seeds ) do @@ -22,12 +22,6 @@ module Gitlab end end - def uses_unsupported_legacy_trigger? - trigger_request.present? && - trigger_request.trigger.legacy? && - !trigger_request.trigger.supports_legacy_tokens? - end - def branch_exists? strong_memoize(:is_branch) do project.repository.branch_exists?(ref) diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index d4b7444005e..66bead3a416 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -9,7 +9,7 @@ module Gitlab include Chain::Helpers SOURCES = [ - Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime, + Gitlab::Ci::Pipeline::Chain::Config::Content::Bridge, Gitlab::Ci::Pipeline::Chain::Config::Content::Repository, Gitlab::Ci::Pipeline::Chain::Config::Content::ExternalProject, Gitlab::Ci::Pipeline::Chain::Config::Content::Remote, @@ -17,15 +17,14 @@ module Gitlab ].freeze LEGACY_SOURCES = [ - Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime, + Gitlab::Ci::Pipeline::Chain::Config::Content::Bridge, Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyRepository, Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyAutoDevops ].freeze def perform! if config = find_config - # TODO: we should persist config_content - # @pipeline.config_content = config.content + @pipeline.build_pipeline_config(content: config.content) if ci_root_config_content_enabled? @command.config_content = config.content @pipeline.config_source = config.source else @@ -49,11 +48,11 @@ module Gitlab end def sources - if Feature.enabled?(:ci_root_config_content, @command.project, default_enabled: true) - SOURCES - else - LEGACY_SOURCES - end + ci_root_config_content_enabled? ? SOURCES : LEGACY_SOURCES + end + + def ci_root_config_content_enabled? + Feature.enabled?(:ci_root_config_content, @command.project, default_enabled: true) end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb b/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb index e9bcc67de9c..54be789988c 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb @@ -11,7 +11,7 @@ module Gitlab strong_memoize(:content) do next unless project&.auto_devops_enabled? - template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps') + template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name) YAML.dump('include' => [{ 'template' => template.full_name }]) end end @@ -19,6 +19,22 @@ module Gitlab def source :auto_devops_source end + + private + + def template_name + if beta_enabled? + 'Beta/Auto-DevOps' + else + 'Auto-DevOps' + end + end + + def beta_enabled? + Feature.enabled?(:auto_devops_beta, project, default_enabled: true) && + # workflow:rules are required by `Beta/Auto-DevOps.gitlab-ci.yml` + Feature.enabled?(:workflow_rules, project, default_enabled: true) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb b/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb new file mode 100644 index 00000000000..39ffa2d4e25 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class Bridge < Source + def content + return unless @command.bridge + + @command.bridge.yaml_for_downstream + end + + def source + :bridge_source + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb b/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb index c4cef356628..b282886a56f 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb @@ -11,7 +11,7 @@ module Gitlab strong_memoize(:content) do next unless project&.auto_devops_enabled? - template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps') + template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name) template.content end end @@ -19,6 +19,22 @@ module Gitlab def source :auto_devops_source end + + private + + def template_name + if beta_enabled? + 'Beta/Auto-DevOps' + else + 'Auto-DevOps' + end + end + + def beta_enabled? + Feature.enabled?(:auto_devops_beta, project, default_enabled: true) && + # workflow:rules are required by `Beta/Auto-DevOps.gitlab-ci.yml` + Feature.enabled?(:workflow_rules, project, default_enabled: true) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/runtime.rb b/lib/gitlab/ci/pipeline/chain/config/content/runtime.rb deleted file mode 100644 index 4811d3d913d..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/runtime.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class Runtime < Source - def content - @command.config_content - end - - def source - # The only case when this source is used is when the config content - # is passed in as parameter to Ci::CreatePipelineService. - # This would only occur with parent/child pipelines which is being - # implemented. - # TODO: change source to return :runtime_source - # https://gitlab.com/gitlab-org/gitlab/merge_requests/21041 - - nil - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index f9ed9d91177..a30b6c6ef0e 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -14,16 +14,12 @@ module Gitlab return error('Pipelines are disabled!') end - if @command.uses_unsupported_legacy_trigger? - return error('Trigger token is invalid because is not owned by any user') + unless allowed_to_create_pipeline? + return error('Insufficient permissions to create a new pipeline') end - unless allowed_to_trigger_pipeline? - if can?(current_user, :create_pipeline, project) - return error("Insufficient permissions for protected ref '#{command.ref}'") - else - return error('Insufficient permissions to create a new pipeline') - end + unless allowed_to_write_ref? + return error("Insufficient permissions for protected ref '#{command.ref}'") end end @@ -31,17 +27,13 @@ module Gitlab @pipeline.errors.any? end - def allowed_to_trigger_pipeline? - if current_user - allowed_to_create? - else # legacy triggers don't have a corresponding user - !@command.protected_ref? - end - end + private - def allowed_to_create? - return unless can?(current_user, :create_pipeline, project) + def allowed_to_create_pipeline? + can?(current_user, :create_pipeline, project) + end + def allowed_to_write_ref? access = Gitlab::UserAccess.new(current_user, project: project) if @command.branch_exists? diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 590c7f4d1dd..98b4b4593e0 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -18,6 +18,7 @@ module Gitlab @seed_attributes = attributes @previous_stages = previous_stages @needs_attributes = dig(:needs_attributes) + @resource_group_key = attributes.delete(:resource_group_key) @using_rules = attributes.key?(:rules) @using_only = attributes.key?(:only) @@ -78,6 +79,7 @@ module Gitlab else ::Ci::Build.new(attributes).tap do |job| job.deployment = Seed::Deployment.new(job).to_resource + job.resource_group = Seed::Build::ResourceGroup.new(job, @resource_group_key).to_resource end end end diff --git a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb b/lib/gitlab/ci/pipeline/seed/build/resource_group.rb new file mode 100644 index 00000000000..3bec6d1e8b6 --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/build/resource_group.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Seed + class Build + class ResourceGroup < Seed::Base + include Gitlab::Utils::StrongMemoize + + attr_reader :build, :resource_group_key + + def initialize(build, resource_group_key) + @build = build + @resource_group_key = resource_group_key + end + + def to_resource + return unless Feature.enabled?(:ci_resource_group, build.project, default_enabled: true) + return unless resource_group_key.present? + + resource_group = build.project.resource_groups + .safe_find_or_create_by(key: expanded_resource_group_key) + + resource_group if resource_group.persisted? + end + + private + + def expanded_resource_group_key + strong_memoize(:expanded_resource_group_key) do + ExpandVariables.expand(resource_group_key, -> { build.simple_variables }) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/snippets/review_app_default.yml b/lib/gitlab/ci/snippets/review_app_default.yml new file mode 100644 index 00000000000..b6db08ef537 --- /dev/null +++ b/lib/gitlab/ci/snippets/review_app_default.yml @@ -0,0 +1,9 @@ +deploy_review: + stage: deploy + script: + - echo "Deploy a review app" + environment: + name: review/$CI_COMMIT_REF_NAME + url: https://$CI_ENVIRONMENT_SLUG.example.com + only: + - branches diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 96d05842838..7e5afbad806 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -11,6 +11,7 @@ module Gitlab Status::Build::Manual, Status::Build::Canceled, Status::Build::Created, + Status::Build::WaitingForResource, Status::Build::Preparing, Status::Build::Pending, Status::Build::Skipped], diff --git a/lib/gitlab/ci/status/build/waiting_for_resource.rb b/lib/gitlab/ci/status/build/waiting_for_resource.rb new file mode 100644 index 00000000000..008e6a17bdd --- /dev/null +++ b/lib/gitlab/ci/status/build/waiting_for_resource.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Build + class WaitingForResource < Status::Extended + ## + # TODO: image is shared with 'pending' + # until we get a dedicated one + # + def illustration + { + image: 'illustrations/pending_job_empty.svg', + size: 'svg-430', + title: _('This job is waiting for resource: ') + subject.resource_group.key + } + end + + def self.matches?(build, _) + build.waiting_for_resource? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 3c00b67911f..074651f1040 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -25,6 +25,8 @@ module Gitlab # 2. In other cases we assume that status is of that type # based on what statuses are no longer valid based on the # data set that we have + # rubocop: disable Metrics/CyclomaticComplexity + # rubocop: disable Metrics/PerceivedComplexity def status return if none? @@ -43,6 +45,8 @@ module Gitlab 'pending' elsif any_of?(:running, :pending) 'running' + elsif any_of?(:waiting_for_resource) + 'waiting_for_resource' elsif any_of?(:manual) 'manual' elsif any_of?(:scheduled) @@ -56,6 +60,8 @@ module Gitlab end end end + # rubocop: enable Metrics/CyclomaticComplexity + # rubocop: enable Metrics/PerceivedComplexity def warnings? @status_set.include?(:success_with_warnings) diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index c29dc51f076..73c73a3b3fc 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -20,7 +20,7 @@ module Gitlab def core_status Gitlab::Ci::Status - .const_get(@status.capitalize, false) + .const_get(@status.to_s.camelize, false) .new(@subject, @user) .extend(self.class.common_helpers) end diff --git a/lib/gitlab/ci/status/waiting_for_resource.rb b/lib/gitlab/ci/status/waiting_for_resource.rb new file mode 100644 index 00000000000..4c9e706bc51 --- /dev/null +++ b/lib/gitlab/ci/status/waiting_for_resource.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + class WaitingForResource < Status::Core + def text + s_('CiStatusText|waiting') + end + + def label + s_('CiStatusLabel|waiting for resource') + end + + def icon + 'status_pending' + end + + def favicon + 'favicon_pending' + end + + def group + 'waiting-for-resource' + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Beta/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Beta/Auto-DevOps.gitlab-ci.yml new file mode 100644 index 00000000000..2c5035705ac --- /dev/null +++ b/lib/gitlab/ci/templates/Beta/Auto-DevOps.gitlab-ci.yml @@ -0,0 +1,163 @@ +# Auto DevOps - BETA do not use +# This CI/CD configuration provides a standard pipeline for +# * building a Docker image (using a buildpack if necessary), +# * storing the image in the container registry, +# * running tests from a buildpack, +# * running code quality analysis, +# * creating a review app for each topic branch, +# * and continuous deployment to production +# +# Test jobs may be disabled by setting environment variables: +# * test: TEST_DISABLED +# * code_quality: CODE_QUALITY_DISABLED +# * license_management: LICENSE_MANAGEMENT_DISABLED +# * performance: PERFORMANCE_DISABLED +# * sast: SAST_DISABLED +# * dependency_scanning: DEPENDENCY_SCANNING_DISABLED +# * container_scanning: CONTAINER_SCANNING_DISABLED +# * dast: DAST_DISABLED +# * review: REVIEW_DISABLED +# * stop_review: REVIEW_DISABLED +# +# In order to deploy, you must have a Kubernetes cluster configured either +# via a project integration, or via group/project variables. +# KUBE_INGRESS_BASE_DOMAIN must also be set on the cluster settings, +# as a variable at the group or project level, or manually added below. +# +# Continuous deployment to production is enabled by default. +# If you want to deploy to staging first, set STAGING_ENABLED environment variable. +# If you want to enable incremental rollout, either manual or time based, +# set INCREMENTAL_ROLLOUT_MODE environment variable to "manual" or "timed". +# If you want to use canary deployments, set CANARY_ENABLED environment variable. +# +# If Auto DevOps fails to detect the proper buildpack, or if you want to +# specify a custom buildpack, set a project variable `BUILDPACK_URL` to the +# repository URL of the buildpack. +# e.g. BUILDPACK_URL=https://github.com/heroku/heroku-buildpack-ruby.git#v142 +# If you need multiple buildpacks, add a file to your project called +# `.buildpacks` that contains the URLs, one on each line, in order. +# Note: Auto CI does not work with multiple buildpacks yet + +image: alpine:latest + +variables: + # KUBE_INGRESS_BASE_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. + # KUBE_INGRESS_BASE_DOMAIN: domain.example.com + + POSTGRES_USER: user + POSTGRES_PASSWORD: testing-password + POSTGRES_ENABLED: "true" + POSTGRES_DB: $CI_ENVIRONMENT_SLUG + POSTGRES_VERSION: 9.6.2 + + DOCKER_DRIVER: overlay2 + + ROLLOUT_RESOURCE_TYPE: deployment + + DOCKER_TLS_CERTDIR: "" # https://gitlab.com/gitlab-org/gitlab-runner/issues/4501 + +stages: + - build + - test + - deploy # dummy stage to follow the template guidelines + - review + - dast + - staging + - canary + - production + - incremental rollout 10% + - incremental rollout 25% + - incremental rollout 50% + - incremental rollout 100% + - performance + - cleanup + +workflow: + rules: + - if: '$BUILDPACK_URL || $AUTO_DEVOPS_EXPLICITLY_ENABLED == "1"' + + - exists: + - Dockerfile + + # https://github.com/heroku/heroku-buildpack-clojure + - exists: + - project.clj + + # https://github.com/heroku/heroku-buildpack-go + - exists: + - go.mod + - Gopkg.mod + - Godeps/Godeps.json + - vendor/vendor.json + - glide.yaml + - src/**/*.go + + # https://github.com/heroku/heroku-buildpack-gradle + - exists: + - gradlew + - build.gradle + - settings.gradle + + # https://github.com/heroku/heroku-buildpack-java + - exists: + - pom.xml + - pom.atom + - pom.clj + - pom.groovy + - pom.rb + - pom.scala + - pom.yaml + - pom.yml + + # https://github.com/heroku/heroku-buildpack-multi + - exists: + - .buildpacks + + # https://github.com/heroku/heroku-buildpack-nodejs + - exists: + - package.json + + # https://github.com/heroku/heroku-buildpack-php + - exists: + - composer.json + - index.php + + # https://github.com/heroku/heroku-buildpack-play + # TODO: detect script excludes some scala files + - exists: + - '**/conf/application.conf' + + # https://github.com/heroku/heroku-buildpack-python + # TODO: detect script checks that all of these exist, not any + - exists: + - requirements.txt + - setup.py + - Pipfile + + # https://github.com/heroku/heroku-buildpack-ruby + - exists: + - Gemfile + + # https://github.com/heroku/heroku-buildpack-scala + - exists: + - '*.sbt' + - project/*.scala + - .sbt/*.scala + - project/build.properties + + # https://github.com/dokku/buildpack-nginx + - exists: + - .static + +include: + - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml + - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml + - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml + - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml + - template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml + - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml + - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml + - template: Security/License-Management.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 1708984c1cb..8bc60a36ebd 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:0.85.5" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:0.85.6" script: - | if ! docker info &>/dev/null; then diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 7a672f910dd..feedb0994c2 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.6.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.8.3" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 9a5b0f79ecf..93c69772b01 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,16 +1,21 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.3.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.5.0" environment: name: production variables: TILLER_NAMESPACE: gitlab-managed-apps GITLAB_MANAGED_APPS_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/config.yaml INGRESS_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/ingress/values.yaml + CERT_MANAGER_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/cert-manager/values.yaml SENTRY_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/sentry/values.yaml + GITLAB_RUNNER_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/gitlab-runner/values.yaml script: - - kubectl get namespace "$TILLER_NAMESPACE" || kubectl create namespace "$TILLER_NAMESPACE" - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml only: refs: - master + artifacts: + when: on_failure + paths: + - tiller.log diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 23c65a0cb67..94b9d94fd39 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -10,10 +10,13 @@ stages: - deploy - dast +variables: + DAST_VERSION: 1 + dast: stage: dast image: - name: "registry.gitlab.com/gitlab-org/security-products/dast:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" + name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" variables: # URL to scan: # DAST_WEBSITE: https://example.com/ diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index d73f6ccdb3f..225fb7e5606 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -49,6 +49,9 @@ dependency_scanning: DS_PYTHON_VERSION \ DS_PIP_VERSION \ DS_PIP_DEPENDENCY_PATH \ + GEMNASIUM_DB_LOCAL_PATH \ + GEMNASIUM_DB_REMOTE_URL \ + GEMNASIUM_DB_REF_NAME \ PIP_INDEX_URL \ PIP_EXTRA_INDEX_URL \ PIP_REQUIREMENTS_FILE \ @@ -77,6 +80,7 @@ dependency_scanning: services: [] except: variables: + - $DEPENDENCY_SCANNING_DISABLED - $DS_DISABLE_DIND == 'false' script: - /analyzer run @@ -88,8 +92,8 @@ gemnasium-dependency_scanning: only: variables: - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby|javascript|php/ + $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby|javascript|php|\bgo\b/ gemnasium-maven-dependency_scanning: extends: .ds-analyzer diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 34d84138a8b..864e3eb569d 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -55,6 +55,7 @@ sast: services: [] except: variables: + - $SAST_DISABLED - $SAST_DISABLE_DIND == 'false' script: - /analyzer run diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 941f7178dac..4e83826b249 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -9,6 +9,10 @@ module Gitlab LOCK_TTL = 10.minutes LOCK_RETRIES = 2 LOCK_SLEEP = 0.001.seconds + WATCH_FLAG_TTL = 10.seconds + + UPDATE_FREQUENCY_DEFAULT = 30.seconds + UPDATE_FREQUENCY_WHEN_BEING_WATCHED = 3.seconds ArchiveError = Class.new(StandardError) AlreadyArchivedError = Class.new(StandardError) @@ -119,6 +123,22 @@ module Gitlab end end + def update_interval + being_watched? ? UPDATE_FREQUENCY_WHEN_BEING_WATCHED : UPDATE_FREQUENCY_DEFAULT + end + + def being_watched! + Gitlab::Redis::SharedState.with do |redis| + redis.set(being_watched_cache_key, true, ex: WATCH_FLAG_TTL) + end + end + + def being_watched? + Gitlab::Redis::SharedState.with do |redis| + redis.exists(being_watched_cache_key) + end + end + private def unsafe_write!(mode, &blk) @@ -236,6 +256,10 @@ module Gitlab def trace_artifact job.job_artifacts_trace end + + def being_watched_cache_key + "gitlab:ci:trace:#{job.id}:watched" + end end end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 27cd4f5fd6b..080a8ac107d 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -64,6 +64,7 @@ module Gitlab except: job[:except], rules: job[:rules], cache: job[:cache], + resource_group_key: job[:resource_group], options: { image: job[:image], services: job[:services], @@ -80,10 +81,15 @@ module Gitlab instance: job[:instance], start_in: job[:start_in], trigger: job[:trigger], - bridge_needs: job.dig(:needs, :bridge)&.first + bridge_needs: job.dig(:needs, :bridge)&.first, + release: release(job) }.compact }.compact end + def release(job) + job[:release] if Feature.enabled?(:ci_release_generation, default_enabled: false) + end + def stage_builds_attributes(stage) @jobs.values .select { |job| job[:stage] == stage } @@ -132,7 +138,6 @@ module Gitlab @jobs.each do |name, job| # logical validation for job - validate_job_stage!(name, job) validate_job_dependencies!(name, job) validate_job_needs!(name, job) diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index 4ba921569ad..8e624215065 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -11,11 +11,13 @@ module Gitlab end def initialize(project, current_user = nil) + @project = project @extractor = Gitlab::ReferenceExtractor.new(project, current_user) end def closed_by_message(message) return [] if message.nil? + return [] unless @project.autoclose_referenced_issues closing_statements = [] message.scan(ISSUE_CLOSING_REGEX) do diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index 2b3dc94fc5e..4ae75e0db0a 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -149,10 +149,10 @@ module Gitlab def in_clustered_environment? # Sidekiq doesn't fork - return false if Sidekiq.server? + return false if Gitlab::Runtime.sidekiq? # Unicorn always forks - return true if defined?(::Unicorn) + return true if Gitlab::Runtime.unicorn? # Puma sometimes forks return true if in_clustered_puma? @@ -162,7 +162,7 @@ module Gitlab end def in_clustered_puma? - return false unless defined?(::Puma) + return false unless Gitlab::Runtime.puma? @puma_options && @puma_options[:workers] && @puma_options[:workers] > 0 end diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb index 87bd257f69a..4deb233d10e 100644 --- a/lib/gitlab/config/entry/attributable.rb +++ b/lib/gitlab/config/entry/attributable.rb @@ -10,7 +10,7 @@ module Gitlab def attributes(*attributes) attributes.flatten.each do |attribute| if method_defined?(attribute) - raise ArgumentError, 'Method already defined!' + raise ArgumentError, "Method already defined: #{attribute}" end define_method(attribute) do diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb index d5a093a469a..e7d441bb21c 100644 --- a/lib/gitlab/config/entry/configurable.rb +++ b/lib/gitlab/config/entry/configurable.rb @@ -5,7 +5,7 @@ module Gitlab module Entry ## # This mixin is responsible for adding DSL, which purpose is to - # simplifly process of adding child nodes. + # simplify the process of adding child nodes. # # This can be used only if parent node is a configuration entry that # holds a hash as a configuration value, for example: diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb index 6fd7214dce7..d5f2e868606 100644 --- a/lib/gitlab/cycle_analytics/production_stage.rb +++ b/lib/gitlab/cycle_analytics/production_stage.rb @@ -18,7 +18,7 @@ module Gitlab end def title - s_('CycleAnalyticsStage|Production') + s_('CycleAnalyticsStage|Total') end def legend diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb new file mode 100644 index 00000000000..c0748a4b8e6 --- /dev/null +++ b/lib/gitlab/danger/commit_linter.rb @@ -0,0 +1,232 @@ +# frozen_string_literal: true + +emoji_checker_path = File.expand_path('emoji_checker', __dir__) +defined?(Rails) ? require_dependency(emoji_checker_path) : require_relative(emoji_checker_path) + +module Gitlab + module Danger + class CommitLinter + MIN_SUBJECT_WORDS_COUNT = 3 + MAX_LINE_LENGTH = 72 + WARN_SUBJECT_LENGTH = 50 + URL_LIMIT_SUBJECT = "https://chris.beams.io/posts/git-commit/#limit-50" + MAX_CHANGED_FILES_IN_COMMIT = 3 + MAX_CHANGED_LINES_IN_COMMIT = 30 + SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(#|!|&|%)\d+\b}.freeze + DEFAULT_SUBJECT_DESCRIPTION = 'commit subject' + PROBLEMS = { + subject_too_short: "The %s must contain at least #{MIN_SUBJECT_WORDS_COUNT} words", + subject_too_long: "The %s may not be longer than #{MAX_LINE_LENGTH} characters", + subject_above_warning: "The %s length is acceptable, but please try to [reduce it to #{WARN_SUBJECT_LENGTH} characters](#{URL_LIMIT_SUBJECT}).", + subject_starts_with_lowercase: "The %s must start with a capital letter", + subject_ends_with_a_period: "The %s must not end with a period", + separator_missing: "The commit subject and body must be separated by a blank line", + details_too_many_changes: "Commits that change #{MAX_CHANGED_LINES_IN_COMMIT} or more lines across " \ + "at least #{MAX_CHANGED_FILES_IN_COMMIT} files must describe these changes in the commit body", + details_line_too_long: "The commit body should not contain more than #{MAX_LINE_LENGTH} characters per line", + message_contains_text_emoji: "Avoid the use of Markdown Emoji such as `:+1:`. These add limited value " \ + "to the commit message, and are displayed as plain text outside of GitLab", + message_contains_unicode_emoji: "Avoid the use of Unicode Emoji. These add no value to the commit " \ + "message, and may not be displayed properly everywhere", + message_contains_short_reference: "Use full URLs instead of short references (`gitlab-org/gitlab#123` or " \ + "`!123`), as short references are displayed as plain text outside of GitLab" + }.freeze + + attr_reader :commit, :problems + + def initialize(commit) + @commit = commit + @problems = {} + @linted = false + end + + def fixup? + commit.message.start_with?('fixup!', 'squash!') + end + + def suggestion? + commit.message.start_with?('Apply suggestion to') + end + + def merge? + commit.message.start_with?('Merge branch') + end + + def revert? + commit.message.start_with?('Revert "') + end + + def multi_line? + !details.nil? && !details.empty? + end + + def failed? + problems.any? + end + + def add_problem(problem_key, *args) + @problems[problem_key] = sprintf(PROBLEMS[problem_key], *args) + end + + def lint(subject_description = "commit subject") + return self if @linted + + @linted = true + lint_subject(subject_description) + lint_separator + lint_details + lint_message + + self + end + + def lint_subject(subject_description) + if subject_too_short? + add_problem(:subject_too_short, subject_description) + end + + if subject_too_long? + add_problem(:subject_too_long, subject_description) + elsif subject_above_warning? + add_problem(:subject_above_warning, subject_description) + end + + if subject_starts_with_lowercase? + add_problem(:subject_starts_with_lowercase, subject_description) + end + + if subject_ends_with_a_period? + add_problem(:subject_ends_with_a_period, subject_description) + end + + self + end + + private + + def lint_separator + return self unless separator && !separator.empty? + + add_problem(:separator_missing) + + self + end + + def lint_details + if !multi_line? && many_changes? + add_problem(:details_too_many_changes) + end + + details&.each_line do |line| + line = line.strip + + next unless line_too_long?(line) + + url_size = line.scan(%r((https?://\S+))).sum { |(url)| url.length } # rubocop:disable CodeReuse/ActiveRecord + + # If the line includes a URL, we'll allow it to exceed MAX_LINE_LENGTH characters, but + # only if the line _without_ the URL does not exceed this limit. + next unless line_too_long?(line.length - url_size) + + add_problem(:details_line_too_long) + break + end + + self + end + + def lint_message + if message_contains_text_emoji? + add_problem(:message_contains_text_emoji) + end + + if message_contains_unicode_emoji? + add_problem(:message_contains_unicode_emoji) + end + + if message_contains_short_reference? + add_problem(:message_contains_short_reference) + end + + self + end + + def files_changed + commit.diff_parent.stats[:total][:files] + end + + def lines_changed + commit.diff_parent.stats[:total][:lines] + end + + def many_changes? + files_changed > MAX_CHANGED_FILES_IN_COMMIT && lines_changed > MAX_CHANGED_LINES_IN_COMMIT + end + + def subject + message_parts[0] + end + + def separator + message_parts[1] + end + + def details + message_parts[2] + end + + def line_too_long?(line) + case line + when String + line.length > MAX_LINE_LENGTH + when Integer + line > MAX_LINE_LENGTH + else + raise ArgumentError, "The line argument (#{line}) should be a String or an Integer! #{line.class} given." + end + end + + def subject_too_short? + subject.split(' ').length < MIN_SUBJECT_WORDS_COUNT + end + + def subject_too_long? + line_too_long?(subject) + end + + def subject_above_warning? + subject.length > WARN_SUBJECT_LENGTH + end + + def subject_starts_with_lowercase? + first_char = subject[0] + + first_char.downcase == first_char + end + + def subject_ends_with_a_period? + subject.end_with?('.') + end + + def message_contains_text_emoji? + emoji_checker.includes_text_emoji?(commit.message) + end + + def message_contains_unicode_emoji? + emoji_checker.includes_unicode_emoji?(commit.message) + end + + def message_contains_short_reference? + commit.message.match?(SHORT_REFERENCE_REGEX) + end + + def emoji_checker + @emoji_checker ||= Gitlab::Danger::EmojiChecker.new + end + + def message_parts + @message_parts ||= commit.message.split("\n", 3) + end + end + end +end diff --git a/lib/gitlab/danger/emoji_checker.rb b/lib/gitlab/danger/emoji_checker.rb new file mode 100644 index 00000000000..e31a6ae5011 --- /dev/null +++ b/lib/gitlab/danger/emoji_checker.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'json' + +module Gitlab + module Danger + class EmojiChecker + DIGESTS = File.expand_path('../../../fixtures/emojis/digests.json', __dir__) + ALIASES = File.expand_path('../../../fixtures/emojis/aliases.json', __dir__) + + # A regex that indicates a piece of text _might_ include an Emoji. The regex + # alone is not enough, as we'd match `:foo:bar:baz`. Instead, we use this + # regex to save us from having to check for all possible emoji names when we + # know one definitely is not included. + LIKELY_EMOJI = /:[\+a-z0-9_\-]+:/.freeze + + UNICODE_EMOJI_REGEX = %r{( + [\u{1F300}-\u{1F5FF}] | + [\u{1F1E6}-\u{1F1FF}] | + [\u{2700}-\u{27BF}] | + [\u{1F900}-\u{1F9FF}] | + [\u{1F600}-\u{1F64F}] | + [\u{1F680}-\u{1F6FF}] | + [\u{2600}-\u{26FF}] + )}x.freeze + + def initialize + names = JSON.parse(File.read(DIGESTS)).keys + + JSON.parse(File.read(ALIASES)).keys + + @emoji = names.map { |name| ":#{name}:" } + end + + def includes_text_emoji?(text) + return false unless text.match?(LIKELY_EMOJI) + + @emoji.any? { |emoji| text.include?(emoji) } + end + + def includes_unicode_emoji?(text) + text.match?(UNICODE_EMOJI_REGEX) + end + end + end +end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 90cef384a1b..5363533ace5 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -174,6 +174,10 @@ module Gitlab labels - current_mr_labels end + def sanitize_mr_title(title) + title.gsub(/^WIP: */, '').gsub(/`/, '\\\`') + end + def security_mr? return false unless gitlab_helper diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index ceab9322857..82ec740ade1 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -241,6 +241,14 @@ module Gitlab row['version'] end + def self.exists? + connection + + true + rescue + false + end + private_class_method :database_version def self.add_post_migrate_path_to_rails(force: false) diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index f9340b262e5..b7d510c19f9 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -158,7 +158,7 @@ module Gitlab # name - The name of the foreign key. # # rubocop:disable Gitlab/RailsLogger - def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, name: nil) + def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, name: nil, validate: true) # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. if transaction_open? @@ -197,14 +197,32 @@ module Gitlab # Validate the existing constraint. This can potentially take a very # long time to complete, but fortunately does not lock the source table # while running. + # Disable this check by passing `validate: false` to the method call + # The check will be enforced for new data (inserts) coming in, + # but validating existing data is delayed. # # Note this is a no-op in case the constraint is VALID already - disable_statement_timeout do - execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{options[:name]};") + + if validate + disable_statement_timeout do + execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{options[:name]};") + end end end # rubocop:enable Gitlab/RailsLogger + def validate_foreign_key(source, column, name: nil) + fk_name = name || concurrent_foreign_key_name(source, column) + + unless foreign_key_exists?(source, name: fk_name) + raise "cannot find #{fk_name} on #{source} table" + end + + disable_statement_timeout do + execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{fk_name};") + end + end + def foreign_key_exists?(source, target = nil, **options) foreign_keys(source).any? do |foreign_key| tables_match?(target.to_s, foreign_key.to_table.to_s) && diff --git a/lib/gitlab/database_importers/instance_administrators/create_group.rb b/lib/gitlab/database_importers/instance_administrators/create_group.rb new file mode 100644 index 00000000000..5bf0e5a320d --- /dev/null +++ b/lib/gitlab/database_importers/instance_administrators/create_group.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module InstanceAdministrators + class CreateGroup < ::BaseService + include Stepable + + VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL + + steps :validate_application_settings, + :validate_admins, + :create_group, + :save_group_id, + :add_group_members, + :track_event + + def initialize + super(nil) + end + + def execute + execute_steps + end + + private + + def validate_application_settings(result) + return success(result) if application_settings + + log_error('No application_settings found') + error(_('No application_settings found')) + end + + def validate_admins(result) + unless instance_admins.any? + log_error('No active admin user found') + return error(_('No active admin user found')) + end + + success(result) + end + + def create_group(result) + if group_created? + log_info(_('Instance administrators group already exists')) + result[:group] = instance_administrators_group + return success(result) + end + + result[:group] = ::Groups::CreateService.new(instance_admins.first, create_group_params).execute + + if result[:group].persisted? + success(result) + else + log_error("Could not create instance administrators group. Errors: %{errors}" % { errors: result[:group].errors.full_messages }) + error(_('Could not create group')) + end + end + + def save_group_id(result) + return success(result) if group_created? + + response = application_settings.update( + instance_administrators_group_id: result[:group].id + ) + + if response + success(result) + else + log_error("Could not save instance administrators group ID, errors: %{errors}" % { errors: application_settings.errors.full_messages }) + error(_('Could not save group ID')) + end + end + + def add_group_members(result) + group = result[:group] + members = group.add_users(members_to_add(group), Gitlab::Access::MAINTAINER) + errors = members.flat_map { |member| member.errors.full_messages } + + if errors.any? + log_error('Could not add admins as members to self-monitoring project. Errors: %{errors}' % { errors: errors }) + error(_('Could not add admins as members')) + else + success(result) + end + end + + def track_event(result) + ::Gitlab::Tracking.event("instance_administrators_group", "group_created") + + success(result) + end + + def group_created? + instance_administrators_group.present? + end + + def application_settings + @application_settings ||= ApplicationSetting.current_without_cache + end + + def instance_administrators_group + application_settings.instance_administrators_group + end + + def instance_admins + @instance_admins ||= User.admins.active + end + + def members_to_add(group) + # Exclude admins who are already members of group because + # `group.add_users(users)` returns an error if the users parameter contains + # users who are already members of the group. + instance_admins - group.members.collect(&:user) + end + + def create_group_params + { + name: 'GitLab Instance Administrators', + visibility_level: VISIBILITY_LEVEL, + + # The 8 random characters at the end are so that the path does not + # clash with any existing group that the user might have created. + path: "gitlab-instance-administrators-#{SecureRandom.hex(4)}" + } + end + end + end + end +end diff --git a/lib/gitlab/database_importers/self_monitoring/helpers.rb b/lib/gitlab/database_importers/self_monitoring/helpers.rb new file mode 100644 index 00000000000..d7e90967e89 --- /dev/null +++ b/lib/gitlab/database_importers/self_monitoring/helpers.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module SelfMonitoring + module Helpers + def application_settings + @application_settings ||= ApplicationSetting.current_without_cache + end + + def project_created? + self_monitoring_project.present? + end + + def self_monitoring_project + application_settings.instance_administration_project + end + + def self_monitoring_project_id + application_settings.instance_administration_project_id + end + end + end + end +end diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb index fbf252b7ec3..d08afeef3b6 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -6,38 +6,24 @@ module Gitlab module Project class CreateService < ::BaseService include Stepable - - STEPS_ALLOWED_TO_FAIL = [ - :validate_application_settings, :validate_project_created, :validate_admins - ].freeze + include SelfMonitoring::Helpers VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL PROJECT_NAME = 'GitLab Instance Administration' steps :validate_application_settings, - :validate_project_created, - :validate_admins, :create_group, :create_project, :save_project_id, - :add_group_members, - :add_prometheus_manual_configuration + :add_prometheus_manual_configuration, + :track_event def initialize super(nil) end - def execute! - result = execute_steps - if result[:status] == :success - ::Gitlab::Tracking.event("self_monitoring", "project_created") - result - elsif STEPS_ALLOWED_TO_FAIL.include?(result[:last_step]) - ::Gitlab::Tracking.event("self_monitoring", "project_created") - success - else - raise StandardError, result[:message] - end + def execute + execute_steps end private @@ -49,46 +35,27 @@ module Gitlab error(_('No application_settings found')) end - def validate_project_created(result) - return success(result) unless project_created? - - log_error('Project already created') - error(_('Project already created')) - end - - def validate_admins(result) - unless instance_admins.any? - log_error('No active admin user found') - return error(_('No active admin user found')) - end - - success(result) - end - def create_group(result) - if project_created? - log_info(_('Instance administrators group already exists')) - result[:group] = application_settings.instance_administration_project.owner - return success(result) - end + create_group_response = + Gitlab::DatabaseImporters::InstanceAdministrators::CreateGroup.new.execute - result[:group] = ::Groups::CreateService.new(group_owner, create_group_params).execute - - if result[:group].persisted? - success(result) + if create_group_response[:status] == :success + success(result.merge(create_group_response)) else - error(_('Could not create group')) + error(create_group_response[:message]) end end def create_project(result) if project_created? log_info('Instance administration project already exists') - result[:project] = application_settings.instance_administration_project + result[:project] = self_monitoring_project return success(result) end - result[:project] = ::Projects::CreateService.new(group_owner, create_project_params(result[:group])).execute + owner = result[:group].owners.first + + result[:project] = ::Projects::CreateService.new(owner, create_project_params(result[:group])).execute if result[:project].persisted? success(result) @@ -99,7 +66,7 @@ module Gitlab end def save_project_id(result) - return success if project_created? + return success(result) if project_created? response = application_settings.update( instance_administration_project_id: result[:project].id @@ -113,19 +80,6 @@ module Gitlab end end - def add_group_members(result) - group = result[:group] - members = group.add_users(members_to_add(group), Gitlab::Access::MAINTAINER) - errors = members.flat_map { |member| member.errors.full_messages } - - if errors.any? - log_error('Could not add admins as members to self-monitoring project. Errors: %{errors}' % { errors: errors }) - error(_('Could not add admins as members')) - else - success(result) - end - end - def add_prometheus_manual_configuration(result) return success(result) unless prometheus_enabled? return success(result) unless prometheus_listen_address.present? @@ -140,12 +94,10 @@ module Gitlab success(result) end - def application_settings - @application_settings ||= ApplicationSetting.current_without_cache - end + def track_event(result) + ::Gitlab::Tracking.event("self_monitoring", "project_created") - def project_created? - application_settings.instance_administration_project.present? + success(result) end def parse_url(uri_string) @@ -161,29 +113,6 @@ module Gitlab ::Gitlab::Prometheus::Internal.listen_address end - def instance_admins - @instance_admins ||= User.admins.active - end - - def group_owner - instance_admins.first - end - - def members_to_add(group) - # Exclude admins who are already members of group because - # `group.add_users(users)` returns an error if the users parameter contains - # users who are already members of the group. - instance_admins - group.members.collect(&:user) - end - - def create_group_params - { - name: 'GitLab Instance Administrators', - path: "gitlab-instance-administrators-#{SecureRandom.hex(4)}", - visibility_level: VISIBILITY_LEVEL - } - end - def docs_path Rails.application.routes.url_helpers.help_page_path( 'administration/monitoring/gitlab_instance_administration_project/index' diff --git a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb b/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb new file mode 100644 index 00000000000..998977b4000 --- /dev/null +++ b/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module SelfMonitoring + module Project + class DeleteService < ::BaseService + include Stepable + include SelfMonitoring::Helpers + + steps :validate_self_monitoring_project_exists, + :destroy_project + + def initialize + super(nil) + end + + def execute + execute_steps + end + + private + + def validate_self_monitoring_project_exists(result) + unless project_created? || self_monitoring_project_id.present? + return error(_('Self monitoring project does not exist')) + end + + success(result) + end + + def destroy_project(result) + return success(result) unless project_created? + + if self_monitoring_project.destroy + success(result) + else + log_error(self_monitoring_project.errors.full_messages) + error(_('Error deleting project. Check logs for error details.')) + end + end + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb index c63d9e5bb71..7af380689d5 100644 --- a/lib/gitlab/dependency_linker.rb +++ b/lib/gitlab/dependency_linker.rb @@ -12,7 +12,8 @@ module Gitlab PodspecJsonLinker, CartfileLinker, GodepsJsonLinker, - RequirementsTxtLinker + RequirementsTxtLinker, + CargoTomlLinker ].freeze def self.linker(blob_name) diff --git a/lib/gitlab/dependency_linker/cargo_toml_linker.rb b/lib/gitlab/dependency_linker/cargo_toml_linker.rb new file mode 100644 index 00000000000..57e0a5f4699 --- /dev/null +++ b/lib/gitlab/dependency_linker/cargo_toml_linker.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module DependencyLinker + class CargoTomlLinker < BaseLinker + self.file_type = :cargo_toml + + def link + return highlighted_text unless toml + + super + end + + private + + def link_dependencies + link_dependencies_at("dependencies") + link_dependencies_at("dev-dependencies") + link_dependencies_at("build-dependencies") + end + + def link_dependencies_at(type) + dependencies = toml[type] + return unless dependencies + + dependencies.each do |name, value| + link_toml(name, value, type) do |name| + "https://crates.io/crates/#{name}" + end + end + end + + def link_toml(key, value, type, &url_proc) + if value.is_a? String + link_regex(/^(?<name>#{key})\s*=\s*"#{value}"/, &url_proc) + else + link_regex(/^\[#{type}\.(?<name>#{key})]/, &url_proc) + end + end + + def toml + @toml ||= TomlRB.parse(plain_text) rescue nil + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 30fe7440148..2ba38f31720 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -358,6 +358,10 @@ module Gitlab end end + def modified_file? + new_file? || deleted_file? || content_changed? + end + # We can't use Object#try because Blob doesn't inherit from Object, but # from BasicObject (via SimpleDelegator). def try_blobs(meth) @@ -393,33 +397,16 @@ module Gitlab end def simple_viewer_class + return DiffViewer::Collapsed if collapsed? return DiffViewer::NotDiffable unless diffable? + return DiffViewer::Text if modified_file? && text? + return DiffViewer::NoPreview if content_changed? + return DiffViewer::Added if new_file? + return DiffViewer::Deleted if deleted_file? + return DiffViewer::Renamed if renamed_file? + return DiffViewer::ModeChanged if mode_changed? - if content_changed? - if text? - DiffViewer::Text - else - DiffViewer::NoPreview - end - elsif new_file? - if text? - DiffViewer::Text - else - DiffViewer::Added - end - elsif deleted_file? - if text? - DiffViewer::Text - else - DiffViewer::Deleted - end - elsif renamed_file? - DiffViewer::Renamed - elsif mode_changed? - DiffViewer::ModeChanged - else - DiffViewer::NoPreview - end + DiffViewer::NoPreview end def rich_viewer_class @@ -427,8 +414,9 @@ module Gitlab end def viewer_class_from(classes) + return if collapsed? return unless diffable? - return unless new_file? || deleted_file? || content_changed? + return unless modified_file? return if different_type? || external_storage_error? verify_binary = !stored_externally? diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index 06cf3d4d168..d27da186de0 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -47,7 +47,7 @@ module Gitlab private def cache - @cache ||= if Feature.enabled?(:hset_redis_diff_caching, project) + @cache ||= if Feature.enabled?(:hset_redis_diff_caching, project, default_enabled: true) Gitlab::Diff::HighlightCache.new(self) else Gitlab::Diff::DeprecatedHighlightCache.new(self) diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb index 3323ce60158..0a14a909e31 100644 --- a/lib/gitlab/email/attachment_uploader.rb +++ b/lib/gitlab/email/attachment_uploader.rb @@ -9,7 +9,7 @@ module Gitlab @message = message end - def execute(project) + def execute(upload_parent:, uploader_class:) attachments = [] message.attachments.each do |attachment| @@ -23,7 +23,7 @@ module Gitlab content_type: attachment.content_type } - uploader = UploadService.new(project, file).execute + uploader = UploadService.new(upload_parent, file, uploader_class).execute attachments << uploader.to_h if uploader ensure tmp.close! diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index d8f4be8ada1..312a9fdfbae 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -41,13 +41,20 @@ module Gitlab end def add_attachments(reply) - attachments = Email::AttachmentUploader.new(mail).execute(project) + attachments = Email::AttachmentUploader.new(mail).execute(upload_params) reply + attachments.map do |link| "\n\n#{link[:markdown]}" end.join end + def upload_params + { + upload_parent: project, + uploader_class: FileUploader + } + end + def validate_permission!(permission) raise UserNotFoundError unless author raise UserBlockedError if author.blocked? @@ -79,3 +86,5 @@ module Gitlab end end end + +Gitlab::Email::Handler::ReplyProcessing.prepend_if_ee('::EE::Gitlab::Email::Handler::ReplyProcessing') diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 847260b2e0f..f028102da9b 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -66,7 +66,8 @@ module Gitlab def key_from_additional_headers(mail) find_key_from_references(mail) || - find_key_from_delivered_to_header(mail) + find_key_from_delivered_to_header(mail) || + find_key_from_envelope_to_header(mail) end def ensure_references_array(references) @@ -96,6 +97,13 @@ module Gitlab end end + def find_key_from_envelope_to_header(mail) + Array(mail[:envelope_to]).find do |header| + key = Gitlab::IncomingEmail.key_from_address(header.value) + break key if key + end + end + def ignore_auto_reply!(mail) if auto_submitted?(mail) || auto_replied?(mail) raise AutoGeneratedEmailError diff --git a/lib/gitlab/error_tracking/detailed_error.rb b/lib/gitlab/error_tracking/detailed_error.rb index 169d6c03f12..c240ec1fa4f 100644 --- a/lib/gitlab/error_tracking/detailed_error.rb +++ b/lib/gitlab/error_tracking/detailed_error.rb @@ -12,10 +12,13 @@ module Gitlab :external_url, :first_release_last_commit, :first_release_short_version, + :first_release_version, :first_seen, :frequency, - :gitlab_project, + :gitlab_commit, + :gitlab_commit_path, :gitlab_issue, + :gitlab_project, :id, :last_release_last_commit, :last_release_short_version, @@ -26,6 +29,7 @@ module Gitlab :project_slug, :short_id, :status, + :tags, :title, :type, :user_count diff --git a/lib/gitlab/error_tracking/repo.rb b/lib/gitlab/error_tracking/repo.rb new file mode 100644 index 00000000000..50611943bac --- /dev/null +++ b/lib/gitlab/error_tracking/repo.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class Repo + attr_accessor :status, :integration_id, :project_id + + def initialize(status:, integration_id:, project_id:) + @status = status + @integration_id = integration_id + @project_id = project_id + end + end + end +end diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb index e0de0219294..92d55213cc2 100644 --- a/lib/gitlab/exception_log_formatter.rb +++ b/lib/gitlab/exception_log_formatter.rb @@ -13,7 +13,7 @@ module Gitlab ) if exception.backtrace - payload['exception.backtrace'] = Gitlab::Profiler.clean_backtrace(exception.backtrace) + payload['exception.backtrace'] = Gitlab::BacktraceCleaner.clean_backtrace(exception.backtrace) end end end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 4fbf15d521a..9d14695c098 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -16,6 +16,12 @@ module Gitlab environment: ::Gitlab.dev_env_or_com?, enabled_ratio: 1, tracking_category: 'Growth::Acquisition::Experiment::SignUpFlow' + }, + paid_signup_flow: { + feature_toggle: :paid_signup_flow, + environment: ::Gitlab.dev_env_or_com?, + enabled_ratio: 0.1, + tracking_category: 'Growth::Acquisition::Experiment::PaidSignUpFlow' } }.freeze diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index a386c21983d..305fbeecce1 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -8,7 +8,7 @@ module Gitlab module FileDetector PATTERNS = { # Project files - readme: /\A(#{Regexp.union(*Gitlab::MarkupHelper::PLAIN_FILENAMES).source})(\.(#{Regexp.union(*Gitlab::MarkupHelper::EXTENSIONS).source}))?\z/i, + readme: /\A(#{Regexp.union(*Gitlab::MarkupHelper::PLAIN_FILENAMES).source})(\.(txt|#{Regexp.union(*Gitlab::MarkupHelper::EXTENSIONS).source}))?\z/i, changelog: %r{\A(changelog|history|changes|news)[^/]*\z}i, license: %r{\A((un)?licen[sc]e|copying)(\.[^/]+)?\z}i, contributing: %r{\Acontributing[^/]*\z}i, @@ -25,6 +25,7 @@ module Gitlab route_map: '.gitlab/route-map.yml', # Dependency files + cargo_toml: 'Cargo.toml', cartfile: %r{\ACartfile[^/]*\z}, composer_json: 'composer.json', gemfile: /\A(Gemfile|gems\.rb)\z/, diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index ec9d2df613b..a71baadfdb3 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -37,7 +37,7 @@ module Gitlab def find_by_path(query) search_paths(query).map do |path| - Gitlab::Search::FoundBlob.new(blob_path: path, project: project, ref: ref, repository: repository) + Gitlab::Search::FoundBlob.new(blob_path: path, path: path, project: project, ref: ref, repository: repository) end end diff --git a/lib/gitlab/plugin.rb b/lib/gitlab/file_hook.rb index b6700f4733b..f886fd10f53 100644 --- a/lib/gitlab/plugin.rb +++ b/lib/gitlab/file_hook.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab - module Plugin + module FileHook def self.any? plugin_glob.any? { |entry| File.file?(entry) } end @@ -17,7 +17,7 @@ module Gitlab def self.execute_all_async(data) args = files.map { |file| [file, data] } - PluginWorker.bulk_perform_async(args) + FileHookWorker.bulk_perform_async(args) end def self.execute(file, data) diff --git a/lib/gitlab/plugin_logger.rb b/lib/gitlab/file_hook_logger.rb index df3bd56fd2f..c5e69172016 100644 --- a/lib/gitlab/plugin_logger.rb +++ b/lib/gitlab/file_hook_logger.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab - class PluginLogger < Gitlab::Logger + class FileHookLogger < Gitlab::Logger def self.file_name_noext 'plugin' end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 8d13c74dca2..b6bffb11344 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -35,16 +35,6 @@ module Gitlab end end - def committer_hash(email:, name:) - return if email.nil? || name.nil? - - { - email: email, - name: name, - time: Time.now - } - end - def tag_name(ref) ref = ref.to_s if self.tag_ref?(ref) @@ -88,6 +78,7 @@ module Gitlab end def shas_eql?(sha1, sha2) + return true if sha1.nil? && sha2.nil? return false if sha1.nil? || sha2.nil? return false unless sha1.class == sha2.class diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index b2dc9a8a3c8..48da838366f 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -18,7 +18,7 @@ module Gitlab :committed_date, :committer_name, :committer_email ].freeze - attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator + attr_accessor(*SERIALIZE_KEYS) def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) @@ -254,7 +254,7 @@ module Gitlab end def no_commit_message - "--no commit message" + "No commit message" end def to_hash diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb index 575e12390cd..92940c352d3 100644 --- a/lib/gitlab/git/gitmodules_parser.rb +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -71,7 +71,7 @@ module Gitlab # Convert from an indexed by name to an array indexed by path # If a submodule doesn't have a path, it is considered bogus # and is ignored - submodules_by_name.each_with_object({}) do |(name, data), results| + submodules_by_name.each_with_object({}) do |(_name, data), results| path = data.delete 'path' next unless path diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 4971a18e270..ed3e7a1e39c 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -853,7 +853,7 @@ module Gitlab end end - def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:, &block) + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:, push_options: [], &block) wrapped_gitaly_errors do gitaly_operation_client.rebase( user, @@ -862,6 +862,7 @@ module Gitlab branch_sha: branch_sha, remote_repository: remote_repository, remote_branch: remote_branch, + push_options: push_options, &block ) end diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb index 80b75689334..068aaf03c51 100644 --- a/lib/gitlab/git/rugged_impl/use_rugged.rb +++ b/lib/gitlab/git/rugged_impl/use_rugged.rb @@ -8,9 +8,17 @@ module Gitlab feature = Feature.get(feature_key) return feature.enabled? if Feature.persisted?(feature) + # Disable Rugged auto-detect(can_use_disk?) when Puma threads>1 + # https://gitlab.com/gitlab-org/gitlab/issues/119326 + return false if running_puma_with_multiple_threads? + Gitlab::GitalyClient.can_use_disk?(repo.storage) end + def running_puma_with_multiple_threads? + Gitlab::Runtime.puma? && ::Puma.cli_config.options[:max_threads] > 1 + end + def execute_rugged_call(method_name, *args) Gitlab::GitalyClient::StorageSettings.allow_disk_access do start = Gitlab::Metrics::System.monotonic_time @@ -27,7 +35,7 @@ module Gitlab feature: method_name, args: args, duration: duration, - backtrace: Gitlab::Profiler.clean_backtrace(caller)) + backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller)) end result diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index 0218f6e6232..08dbd52e3fb 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -10,7 +10,7 @@ module Gitlab MAX_TAG_MESSAGE_DISPLAY_SIZE = 10.megabytes SERIALIZE_KEYS = %i[name target target_commit message].freeze - attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator + attr_accessor(*SERIALIZE_KEYS) class << self def get_message(repository, tag_id) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 9e033c705bd..262a1ef653f 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -29,7 +29,7 @@ module Gitlab PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 30 - CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze + CLIENT_NAME = (Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze GITALY_METADATA_FILENAME = '.gitaly-metadata' MUTEX = Mutex.new @@ -160,6 +160,7 @@ module Gitlab def self.execute(storage, service, rpc, request, remote_storage:, timeout:) enforce_gitaly_request_limits(:call) + Gitlab::RequestContext.instance.ensure_deadline_not_exceeded! kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage) kwargs = yield(kwargs) if block_given? @@ -179,7 +180,7 @@ module Gitlab self.query_time += duration if Gitlab::PerformanceBar.enabled_for_request? add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc, - backtrace: Gitlab::Profiler.clean_backtrace(caller)) + backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller)) end end @@ -234,12 +235,28 @@ module Gitlab metadata['gitaly-session-id'] = session_id metadata.merge!(Feature::Gitaly.server_feature_flags) - result = { metadata: metadata } + deadline_info = request_deadline(timeout) + metadata.merge!(deadline_info.slice(:deadline_type)) - result[:deadline] = real_time + timeout if timeout > 0 - result + { metadata: metadata, deadline: deadline_info[:deadline] } end + def self.request_deadline(timeout) + # timeout being 0 means the request is allowed to run indefinitely. + # We can't allow that inside a request, but this won't count towards Gitaly + # error budgets + regular_deadline = real_time.to_i + timeout if timeout > 0 + + return { deadline: regular_deadline } if Sidekiq.server? + return { deadline: regular_deadline } unless Gitlab::RequestContext.instance.request_deadline + + limited_deadline = [regular_deadline, Gitlab::RequestContext.instance.request_deadline].compact.min + limited = limited_deadline < regular_deadline + + { deadline: limited_deadline, deadline_type: limited ? "limited" : "regular" } + end + private_class_method :request_deadline + def self.session_id Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid end @@ -382,17 +399,13 @@ module Gitlab end def self.long_timeout - if web_app_server? + if Gitlab::Runtime.web_server? default_timeout else 6.hours end end - def self.web_app_server? - defined?(::Unicorn) || defined?(::Puma) - end - def self.storage_metadata_file_path(storage) Gitlab::GitalyClient::StorageSettings.allow_disk_access do File.join( @@ -442,7 +455,7 @@ module Gitlab def self.count_stack return unless Gitlab::SafeRequestStore.active? - stack_string = Gitlab::Profiler.clean_backtrace(caller).drop(1).join("\n") + stack_string = Gitlab::BacktraceCleaner.clean_backtrace(caller).drop(1).join("\n") Gitlab::SafeRequestStore[:stack_counter] ||= Hash.new diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 61c5db4c4df..27522f89a5b 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -233,7 +233,7 @@ module Gitlab end end - def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:, push_options: []) request_enum = QueueEnumerator.new rebase_sha = nil @@ -256,7 +256,8 @@ module Gitlab branch: encode_binary(branch), branch_sha: branch_sha, remote_repository: remote_repository.gitaly_repository, - remote_branch: encode_binary(remote_branch) + remote_branch: encode_binary(remote_branch), + git_push_options: push_options ) ) ) diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 826b35d685c..22803c5cd71 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -189,7 +189,7 @@ module Gitlab end def default_api_endpoint - OmniAuth::Strategies::GitHub.default_options[:client_options][:site] + OmniAuth::Strategies::GitHub.default_options[:client_options][:site] || ::Octokit::Default.api_endpoint end def verify_ssl diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index 6d2aff63a47..f09e0bd9806 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -27,6 +27,7 @@ module Gitlab mr, already_exists = create_merge_request if mr + set_merge_request_assignees(mr) insert_git_data(mr, already_exists) issuable_finder.cache_database_id(mr.id) end @@ -57,7 +58,6 @@ module Gitlab state_id: ::MergeRequest.available_states[pull_request.state], milestone_id: milestone_finder.id_for(pull_request), author_id: author_id, - assignee_id: user_finder.assignee_id_for(pull_request), created_at: pull_request.created_at, updated_at: pull_request.updated_at } @@ -65,6 +65,10 @@ module Gitlab create_merge_request_without_hooks(project, attributes, pull_request.iid) end + def set_merge_request_assignees(merge_request) + merge_request.assignee_ids = [user_finder.assignee_id_for(pull_request)] + end + def insert_git_data(merge_request, already_exists) insert_or_replace_git_data(merge_request, pull_request.source_branch_sha, pull_request.target_branch_sha, already_exists) # We need to create the branch after the merge request is diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index f22c69c531a..2e27e954e79 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -28,6 +28,7 @@ module Gitlab gon.sprite_file_icons = IconsHelper.sprite_file_icons_path gon.emoji_sprites_css_path = ActionController::Base.helpers.stylesheet_path('emoji_sprites') gon.test_env = Rails.env.test? + gon.disable_animations = Gitlab.config.gitlab['disable_animations'] gon.suggested_label_colors = LabelsHelper.suggested_colors gon.first_day_of_week = current_user&.first_day_of_week || Gitlab::CurrentSettings.first_day_of_week gon.ee = Gitlab.ee? diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index e3c474bc0fe..7e6f6a519a6 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -135,7 +135,7 @@ module Gitlab end def cleanup_time - Sidekiq.server? ? BG_CLEANUP_RUNTIME_S : FG_CLEANUP_RUNTIME_S + Gitlab::Runtime.sidekiq? ? BG_CLEANUP_RUNTIME_S : FG_CLEANUP_RUNTIME_S end def tmp_keychains_created diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index 26e8c53032f..94871498cf8 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -40,7 +40,7 @@ module Gitlab def authorize!(object) unless authorized_resource?(object) - raise_resource_not_avaiable_error! + raise_resource_not_available_error! end end @@ -63,7 +63,7 @@ module Gitlab end end - def raise_resource_not_avaiable_error! + def raise_resource_not_available_error! raise Gitlab::Graphql::Errors::ResourceNotAvailable, RESOURCE_ACCESS_ERROR end end diff --git a/lib/gitlab/graphql/connections.rb b/lib/gitlab/graphql/connections.rb index 38c7d98f37c..08d5cd0b72e 100644 --- a/lib/gitlab/graphql/connections.rb +++ b/lib/gitlab/graphql/connections.rb @@ -12,6 +12,10 @@ module Gitlab Gitlab::Graphql::FilterableArray, Gitlab::Graphql::Connections::FilterableArrayConnection ) + GraphQL::Relay::BaseConnection.register_connection_implementation( + Gitlab::Graphql::ExternallyPaginatedArray, + Gitlab::Graphql::Connections::ExternallyPaginatedArrayConnection + ) end end end diff --git a/lib/gitlab/graphql/connections/externally_paginated_array_connection.rb b/lib/gitlab/graphql/connections/externally_paginated_array_connection.rb new file mode 100644 index 00000000000..f0861260691 --- /dev/null +++ b/lib/gitlab/graphql/connections/externally_paginated_array_connection.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +# Make a customized connection type +module Gitlab + module Graphql + module Connections + class ExternallyPaginatedArrayConnection < GraphQL::Relay::ArrayConnection + # As the pagination happens externally + # we just return all the nodes here. + def sliced_nodes + @nodes + end + + def start_cursor + nodes.previous_cursor + end + + def end_cursor + nodes.next_cursor + end + + def next_page? + end_cursor.present? + end + + def previous_page? + start_cursor.present? + end + + alias_method :has_next_page, :next_page? + alias_method :has_previous_page, :previous_page? + end + end + end +end diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb index 41aef64f683..6abd56c89c6 100644 --- a/lib/gitlab/graphql/docs/renderer.rb +++ b/lib/gitlab/graphql/docs/renderer.rb @@ -10,7 +10,7 @@ module Gitlab # It uses graphql-docs helpers and schema parser, more information in https://github.com/gjtorikian/graphql-docs. # # Arguments: - # schema - the GraphQL schema defition. For GitLab should be: GitlabSchema.graphql_definition + # schema - the GraphQL schema definition. For GitLab should be: GitlabSchema.graphql_definition # output_dir: The folder where the markdown files will be saved # template: The path of the haml template to be parsed class Renderer diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index 33acff38ef4..52568286dca 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -9,11 +9,15 @@ The API can be explored interactively using the [GraphiQL IDE](../index.md#graphiql). - ## Objects +Each table below documents a GraphQL type. Types match loosely to models, but not all +fields and methods on a model are available via GraphQL. \ - objects.each do |type| - unless type[:fields].empty? - = "### #{type[:name]}" + = "## #{type[:name]}" + - if type[:description]&.present? + \ + = type[:description] \ ~ "| Name | Type | Description |" ~ "| --- | ---- | ---------- |" diff --git a/lib/gitlab/graphql/externally_paginated_array.rb b/lib/gitlab/graphql/externally_paginated_array.rb new file mode 100644 index 00000000000..4797fe15cd3 --- /dev/null +++ b/lib/gitlab/graphql/externally_paginated_array.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + class ExternallyPaginatedArray < Array + attr_reader :previous_cursor, :next_cursor + + def initialize(previous_cursor, next_cursor, *args) + super(args) + @previous_cursor = previous_cursor + @next_cursor = next_cursor + end + end + end +end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 334642f252e..8597903ad00 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -30,7 +30,7 @@ module Gitlab # rubocop:enable CodeReuse/ActiveRecord def issuable_params - super.merge(group_id: group.id) + super.merge(group_id: group.id, include_subgroups: true) end end end diff --git a/lib/gitlab/health_checks/puma_check.rb b/lib/gitlab/health_checks/puma_check.rb index 7aafe29fbae..9f09070a57d 100644 --- a/lib/gitlab/health_checks/puma_check.rb +++ b/lib/gitlab/health_checks/puma_check.rb @@ -18,7 +18,7 @@ module Gitlab end def check - return unless defined?(::Puma) + return unless Gitlab::Runtime.puma? stats = Puma.stats stats = JSON.parse(stats) diff --git a/lib/gitlab/health_checks/unicorn_check.rb b/lib/gitlab/health_checks/unicorn_check.rb index a30ae015257..cdc6d2a7519 100644 --- a/lib/gitlab/health_checks/unicorn_check.rb +++ b/lib/gitlab/health_checks/unicorn_check.rb @@ -30,7 +30,7 @@ module Gitlab # to change so we can cache the list of servers. def http_servers strong_memoize(:http_servers) do - next unless defined?(::Unicorn::HttpServer) + next unless Gitlab::Runtime.unicorn? ObjectSpace.each_object(::Unicorn::HttpServer).to_a end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 2c243a0d0ae..22b9a038768 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -68,7 +68,7 @@ module Gitlab end def timeout_time - Sidekiq.server? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND + Gitlab::Runtime.sidekiq? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND end def link_dependencies(text, highlighted_text) diff --git a/lib/gitlab/import/merge_request_helpers.rb b/lib/gitlab/import/merge_request_helpers.rb index 4bc39868389..c5694d95aa1 100644 --- a/lib/gitlab/import/merge_request_helpers.rb +++ b/lib/gitlab/import/merge_request_helpers.rb @@ -60,6 +60,7 @@ module Gitlab diff.importing = true diff.save diff.save_git_content + diff.set_as_latest_diff end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 516e7f54a6e..8ce6549c0c7 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -50,6 +50,14 @@ module Gitlab 'VERSION' end + def gitlab_version_filename + 'GITLAB_VERSION' + end + + def gitlab_revision_filename + 'GITLAB_REVISION' + end + def export_filename(exportable:) basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{exportable.full_path.tr('/', '_')}" diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index 00c4c41e6be..d1c20dff799 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -3,7 +3,14 @@ module Gitlab module ImportExport class AttributeCleaner - ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + %w[group_id commit_id discussion_id custom_attributes] + ALLOWED_REFERENCES = [ + *ProjectRelationFactory::PROJECT_REFERENCES, + *ProjectRelationFactory::USER_REFERENCES, + 'group_id', + 'commit_id', + 'discussion_id', + 'custom_attributes' + ].freeze PROHIBITED_REFERENCES = Regexp.union(/\Acached_markdown_version\Z/, /_id\Z/, /_ids\Z/, /_html\Z/, /attributes/).freeze def self.clean(*args) diff --git a/lib/gitlab/import_export/base_object_builder.rb b/lib/gitlab/import_export/base_object_builder.rb new file mode 100644 index 00000000000..ec66b7a7a4f --- /dev/null +++ b/lib/gitlab/import_export/base_object_builder.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + # Base class for Group & Project Object Builders. + # This class is not intended to be used on its own but + # rather inherited from. + # + # Cache keeps 1000 entries at most, 1000 is chosen based on: + # - one cache entry uses around 0.5K memory, 1000 items uses around 500K. + # (leave some buffer it should be less than 1M). It is afforable cost for project import. + # - for projects in Gitlab.com, it seems 1000 entries for labels/milestones is enough. + # For example, gitlab has ~970 labels and 26 milestones. + LRU_CACHE_SIZE = 1000 + + class BaseObjectBuilder + def self.build(*args) + new(*args).find + end + + def initialize(klass, attributes) + @klass = klass.ancestors.include?(Label) ? Label : klass + @attributes = attributes + + if Gitlab::SafeRequestStore.active? + @lru_cache = cache_from_request_store + @cache_key = [klass, attributes] + end + end + + def find + find_with_cache do + find_object || klass.create(prepare_attributes) + end + end + + protected + + def where_clauses + raise NotImplementedError + end + + # attributes wrapped in a method to be + # adjusted in sub-class if needed + def prepare_attributes + attributes + end + + private + + attr_reader :klass, :attributes, :lru_cache, :cache_key + + def find_with_cache + return yield unless lru_cache && cache_key + + lru_cache[cache_key] ||= yield + end + + def cache_from_request_store + Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE) + end + + def find_object + klass.where(where_clause).first + end + + def where_clause + where_clauses.reduce(:and) + end + + def table + @table ||= klass.arel_table + end + + # Returns Arel clause: + # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"` + # from the given Hash of attributes. + def attrs_to_arel(attrs) + attrs.map do |key, value| + table[key].eq(value) + end.reduce(:and) + end + + # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'` + # if attributes has 'title key, otherwise `nil`. + def where_clause_for_title + attrs_to_arel(attributes.slice('title')) + end + + # Returns Arel clause `"{table_name}"."description" = '{attributes['description']}'` + # if attributes has 'description key, otherwise `nil`. + def where_clause_for_description + attrs_to_arel(attributes.slice('description')) + end + + # Returns Arel clause `"{table_name}"."created_at" = '{attributes['created_at']}'` + # if attributes has 'created_at key, otherwise `nil`. + def where_clause_for_created_at + attrs_to_arel(attributes.slice('created_at')) + end + end + end +end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/base_relation_factory.rb index 1438a7db001..562b549f6a1 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/base_relation_factory.rb @@ -2,48 +2,32 @@ module Gitlab module ImportExport - class RelationFactory - prepend_if_ee('::EE::Gitlab::ImportExport::RelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule - - OVERRIDES = { snippets: :project_snippets, - ci_pipelines: 'Ci::Pipeline', - pipelines: 'Ci::Pipeline', - stages: 'Ci::Stage', - statuses: 'commit_status', - triggers: 'Ci::Trigger', - pipeline_schedules: 'Ci::PipelineSchedule', - builds: 'Ci::Build', - runners: 'Ci::Runner', - hooks: 'ProjectHook', - merge_access_levels: 'ProtectedBranch::MergeAccessLevel', - push_access_levels: 'ProtectedBranch::PushAccessLevel', - create_access_levels: 'ProtectedTag::CreateAccessLevel', - labels: :project_labels, - priorities: :label_priorities, - auto_devops: :project_auto_devops, - label: :project_label, - custom_attributes: 'ProjectCustomAttribute', - project_badges: 'Badge', - metrics: 'MergeRequest::Metrics', - ci_cd_settings: 'ProjectCiCdSetting', - error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', - links: 'Releases::Link', - metrics_setting: 'ProjectMetricsSetting' }.freeze - - USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id owner_id].freeze - - PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze - - BUILD_MODELS = %i[Ci::Build commit_status].freeze + class BaseRelationFactory + include Gitlab::Utils::StrongMemoize IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature merge_request ProjectCiCdSetting container_expiration_policy].freeze + OVERRIDES = {}.freeze + EXISTING_OBJECT_RELATIONS = %i[].freeze - TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze + # This represents all relations that have unique key on `project_id` or `group_id` + UNIQUE_RELATIONS = %i[].freeze - # This represents all relations that have unique key on `project_id` - UNIQUE_RELATIONS = %i[project_feature ProjectCiCdSetting container_expiration_policy].freeze + USER_REFERENCES = %w[ + author_id + assignee_id + updated_by_id + merged_by_id + latest_closed_by_id + user_id + created_by_id + last_edited_by_id + merge_user_id + resolved_by_id + closed_by_id owner_id + ].freeze + + TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze def self.create(*args) new(*args).create @@ -58,16 +42,16 @@ module Gitlab relation_name.to_s.constantize end - def initialize(relation_sym:, relation_hash:, members_mapper:, merge_requests_mapping:, user:, project:, excluded_keys: []) + def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, merge_requests_mapping: nil, user:, importable:, excluded_keys: []) @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym @relation_hash = relation_hash.except('noteable_id') @members_mapper = members_mapper + @object_builder = object_builder @merge_requests_mapping = merge_requests_mapping @user = user - @project = project + @importable = importable @imported_object_retries = 0 - - @relation_hash['project_id'] = @project.id + @relation_hash[importable_column_name] = @importable.id # Remove excluded keys from relation_hash # We don't do this in the parsed_relation_hash because of the 'transformed attributes' @@ -82,48 +66,46 @@ module Gitlab # the relation_hash, updating references with new object IDs, mapping users using # the "members_mapper" object, also updating notes if required. def create - return if unknown_service? - - # Do not import legacy triggers - return if !Feature.enabled?(:use_legacy_pipeline_triggers, @project) && legacy_trigger? + return if invalid_relation? + setup_base_models setup_models generate_imported_object end def self.overrides - OVERRIDES + self::OVERRIDES end - def self.existing_object_check - EXISTING_OBJECT_CHECK + def self.existing_object_relations + self::EXISTING_OBJECT_RELATIONS end private + def invalid_relation? + false + end + def setup_models - case @relation_name - when :merge_request_diff_files then setup_diff - when :notes then setup_note - end + raise NotImplementedError + end + + def unique_relations + # define in sub-class if any + self.class::UNIQUE_RELATIONS + end + def setup_base_models update_user_references - update_project_references - update_group_references remove_duplicate_assignees - - if @relation_name == :'Ci::Pipeline' - update_merge_request_references - setup_pipeline - end - reset_tokens! remove_encrypted_attributes! end def update_user_references - USER_REFERENCES.each do |reference| + self.class::USER_REFERENCES.each do |reference| if @relation_hash[reference] @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] end @@ -138,95 +120,14 @@ module Gitlab @relation_hash['issue_assignees'].uniq!(&:user_id) end - def setup_note - set_note_author - # attachment is deprecated and note uploads are handled by Markdown uploader - @relation_hash['attachment'] = nil - end - - # Sets the author for a note. If the user importing the project - # has admin access, an actual mapping with new project members - # will be used. Otherwise, a note stating the original author name - # is left. - def set_note_author - old_author_id = @relation_hash['author_id'] - author = @relation_hash.delete('author') - - update_note_for_missing_author(author['name']) unless has_author?(old_author_id) - end - - def has_author?(old_author_id) - admin_user? && @members_mapper.include?(old_author_id) - end - - def missing_author_note(updated_at, author_name) - timestamp = updated_at.split('.').first - "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" - end - def generate_imported_object - if BUILD_MODELS.include?(@relation_name) - @relation_hash.delete('trace') # old export files have trace - @relation_hash.delete('token') - @relation_hash.delete('commands') - @relation_hash.delete('artifacts_file_store') - @relation_hash.delete('artifacts_metadata_store') - @relation_hash.delete('artifacts_size') - - imported_object - elsif @relation_name == :merge_requests - MergeRequestParser.new(@project, @relation_hash.delete('diff_head_sha'), imported_object, @relation_hash).parse! - else - imported_object - end - end - - def update_project_references - # If source and target are the same, populate them with the new project ID. - if @relation_hash['source_project_id'] - @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID - end - - @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id'] - end - - def same_source_and_target? - @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] - end - - def update_group_references - return unless self.class.existing_object_check.include?(@relation_name) - return unless @relation_hash['group_id'] - - @relation_hash['group_id'] = @project.namespace_id - end - - # This code is a workaround for broken project exports that don't - # export merge requests with CI pipelines (i.e. exports that were - # generated from - # https://gitlab.com/gitlab-org/gitlab/merge_requests/17844). - # This method can be removed in GitLab 12.6. - def update_merge_request_references - # If a merge request was properly created, we don't need to fix - # up this export. - return if @relation_hash['merge_request'] - - merge_request_id = @relation_hash['merge_request_id'] - - return unless merge_request_id - - new_merge_request_id = @merge_requests_mapping[merge_request_id] - - return unless new_merge_request_id - - @relation_hash['merge_request_id'] = new_merge_request_id - parsed_relation_hash['merge_request_id'] = new_merge_request_id + imported_object end def reset_tokens! - return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name) + return unless Gitlab::ImportExport.reset_tokens? && self.class::TOKEN_RESET_MODELS.include?(@relation_name) - # If we import/export a project to the same instance, tokens will have to be reset. + # If we import/export to the same instance, tokens will have to be reset. # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across. relation_class.attribute_names.select { |name| name.include?('token') }.each do |token| @relation_hash[token] = nil @@ -245,6 +146,14 @@ module Gitlab @relation_class ||= self.class.relation_class(@relation_name) end + def importable_column_name + importable_class_name.concat('_id') + end + + def importable_class_name + @importable.class.to_s.downcase + end + def imported_object if existing_or_new_object.respond_to?(:importing) existing_or_new_object.importing = true @@ -258,37 +167,16 @@ module Gitlab retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES end - def update_note_for_missing_author(author_name) - @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? - @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}" - end - - def admin_user? - @user.admin? - end - def parsed_relation_hash @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, relation_class: relation_class) end - def setup_diff - @relation_hash['diff'] = @relation_hash.delete('utf8_diff') - end - - def setup_pipeline - @relation_hash.fetch('stages', []).each do |stage| - stage.statuses.each do |status| - status.pipeline = imported_object - end - end - end - def existing_or_new_object # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. @existing_or_new_object ||= begin - if self.class.existing_object_check.include?(@relation_name) + if existing_object? attribute_hash = attribute_hash_for(['events']) existing_object.assign_attributes(attribute_hash) if attribute_hash.any? @@ -307,7 +195,7 @@ module Gitlab end def attribute_hash_for(attributes) - attributes.inject({}) do |hash, value| + attributes.each_with_object({}) do |hash, value| hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value] hash end @@ -317,31 +205,101 @@ module Gitlab @existing_object ||= find_or_create_object! end - def unknown_service? - @relation_name == :services && parsed_relation_hash['type'] && - !Object.const_defined?(parsed_relation_hash['type']) + def unique_relation_object + unique_relation_object = relation_class.find_or_create_by(importable_column_name => @importable.id) + unique_relation_object.assign_attributes(parsed_relation_hash) + unique_relation_object + end + + def find_or_create_object! + return unique_relation_object if unique_relation? + + # Can't use IDs as validation exists calling `group` or `project` attributes + finder_hash = parsed_relation_hash.tap do |hash| + if relation_class.attribute_method?('group_id') && @importable.is_a?(Project) + hash['group'] = @importable.group + end + + hash[importable_class_name] = @importable if relation_class.reflect_on_association(importable_class_name.to_sym) + hash.delete(importable_column_name) + end + + @object_builder.build(relation_class, finder_hash) end - def legacy_trigger? - @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil? + def setup_note + set_note_author + # attachment is deprecated and note uploads are handled by Markdown uploader + @relation_hash['attachment'] = nil end - def find_or_create_object! - if UNIQUE_RELATIONS.include?(@relation_name) - unique_relation_object = relation_class.find_or_create_by(project_id: @project.id) - unique_relation_object.assign_attributes(parsed_relation_hash) + # Sets the author for a note. If the user importing the project + # has admin access, an actual mapping with new project members + # will be used. Otherwise, a note stating the original author name + # is left. + def set_note_author + old_author_id = @relation_hash['author_id'] + author = @relation_hash.delete('author') + + update_note_for_missing_author(author['name']) unless has_author?(old_author_id) + end + + def has_author?(old_author_id) + admin_user? && @members_mapper.include?(old_author_id) + end + + def missing_author_note(updated_at, author_name) + timestamp = updated_at.split('.').first + "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" + end + + def update_note_for_missing_author(author_name) + @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? + @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}" + end - return unique_relation_object + def admin_user? + @user.admin? + end + + def existing_object? + strong_memoize(:_existing_object) do + self.class.existing_object_relations.include?(@relation_name) || unique_relation? end + end - # Can't use IDs as validation exists calling `group` or `project` attributes - finder_hash = parsed_relation_hash.tap do |hash| - hash['group'] = @project.group if relation_class.attribute_method?('group_id') - hash['project'] = @project if relation_class.reflect_on_association(:project) - hash.delete('project_id') + def unique_relation? + strong_memoize(:unique_relation) do + importable_foreign_key.present? && + (has_unique_index_on_importable_fk? || uses_importable_fk_as_primary_key?) end + end + + def has_unique_index_on_importable_fk? + cache = cached_has_unique_index_on_importable_fk + table_name = relation_class.table_name + return cache[table_name] if cache.has_key?(table_name) + + index_exists = + ActiveRecord::Base.connection.index_exists?( + relation_class.table_name, + importable_foreign_key, + unique: true) + + cache[table_name] = index_exists + end + + # Avoid unnecessary DB requests + def cached_has_unique_index_on_importable_fk + Thread.current[:cached_has_unique_index_on_importable_fk] ||= {} + end + + def uses_importable_fk_as_primary_key? + relation_class.primary_key == importable_foreign_key + end - GroupProjectObjectBuilder.build(relation_class, finder_hash) + def importable_foreign_key + relation_class.reflect_on_association(importable_class_name.to_sym)&.foreign_key end end end diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb index b94839363df..d6d780f165e 100644 --- a/lib/gitlab/import_export/group_project_object_builder.rb +++ b/lib/gitlab/import_export/group_project_object_builder.rb @@ -11,35 +11,29 @@ module Gitlab # finds or initializes a label with the given attributes. # # It also adds some logic around Group Labels/Milestones for edge cases. - class GroupProjectObjectBuilder + class GroupProjectObjectBuilder < BaseObjectBuilder def self.build(*args) Project.transaction do - new(*args).find + super end end def initialize(klass, attributes) - @klass = klass < Label ? Label : klass - @attributes = attributes + super + @group = @attributes['group'] @project = @attributes['project'] end def find - find_object || klass.create(project_attributes) + return if epic? && group.nil? + + super end private - attr_reader :klass, :attributes, :group, :project - - def find_object - klass.where(where_clause).first - end - - def where_clause - where_clauses.reduce(:and) - end + attr_reader :group, :project def where_clauses [ @@ -54,32 +48,18 @@ module Gitlab # or, if group is present: # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}` def where_clause_base - clause = table[:project_id].eq(project.id) if project - clause = clause.or(table[:group_id].eq(group.id)) if group - - clause - end - - # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'` - # if attributes has 'title key, otherwise `nil`. - def where_clause_for_title - attrs_to_arel(attributes.slice('title')) - end - - # Returns Arel clause: - # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"` - # from the given Hash of attributes. - def attrs_to_arel(attrs) - attrs.map do |key, value| - table[key].eq(value) - end.reduce(:and) + [].tap do |clauses| + clauses << table[:project_id].eq(project.id) if project + clauses << table[:group_id].eq(group.id) if group + end.reduce(:or) end - def table - @table ||= klass.arel_table + # Returns Arel clause for a particular model or `nil`. + def where_clause_for_klass + attrs_to_arel(attributes.slice('iid')) if merge_request? end - def project_attributes + def prepare_attributes attributes.except('group').tap do |atts| if label? atts['type'] = 'ProjectLabel' # Always create project labels @@ -108,6 +88,10 @@ module Gitlab klass == MergeRequest end + def epic? + klass == Epic + end + # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: @@ -124,13 +108,6 @@ module Gitlab milestone.ensure_project_iid! milestone.save! end - - protected - - # Returns Arel clause for a particular model or `nil`. - def where_clause_for_klass - return attrs_to_arel(attributes.slice('iid')) if merge_request? - end end end end diff --git a/lib/gitlab/import_export/group_tree_saver.rb b/lib/gitlab/import_export/group_tree_saver.rb index 8d2fb881cc0..2effcd01e30 100644 --- a/lib/gitlab/import_export/group_tree_saver.rb +++ b/lib/gitlab/import_export/group_tree_saver.rb @@ -3,7 +3,7 @@ module Gitlab module ImportExport class GroupTreeSaver - attr_reader :full_path + attr_reader :full_path, :shared def initialize(group:, current_user:, shared:, params: {}) @params = params diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 4f4b4c02eb9..2acb79e3e22 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -16,6 +16,7 @@ tree: - :timelogs - notes: - :author + - :award_emoji - events: - :push_event_payload - label_links: @@ -24,24 +25,30 @@ tree: - milestone: - events: - :push_event_payload + - issue_milestones: + - :milestone - resource_label_events: - label: - :priorities - :issue_assignees - :zoom_meetings - :sentry_issue + - :award_emoji - snippets: - :award_emoji - notes: - :author + - :award_emoji - releases: - :links - project_members: - :user - merge_requests: - :metrics + - :award_emoji - notes: - :author + - :award_emoji - events: - :push_event_payload - :suggestions @@ -57,6 +64,8 @@ tree: - milestone: - events: - :push_event_payload + - merge_request_milestones: + - :milestone - resource_label_events: - label: - :priorities @@ -168,6 +177,8 @@ excluded_attributes: - :secret - :encrypted_secret_token - :encrypted_secret_token_iv + - :repository_storage + - :storage_version merge_request_diff: - :external_diff - :stored_externally @@ -202,6 +213,12 @@ excluded_attributes: - :latest_merge_request_diff_id - :head_pipeline_id - :state_id + issue_milestones: + - :milestone_id + - :issue_id + merge_request_milestones: + - :milestone_id + - :merge_request_id award_emoji: - :awardable_id statuses: @@ -223,6 +240,7 @@ excluded_attributes: - :upstream_pipeline_id - :resource_group_id - :waiting_for_resource_at + - :processed sentry_issue: - :issue_id push_event_payload: @@ -305,6 +323,13 @@ excluded_attributes: - :board_id - :label_id - :milestone_id + epic: + - :start_date_sourcing_milestone_id + - :due_date_sourcing_milestone_id + - :parent_id + - :state_id + - :start_date_sourcing_epic_id + - :due_date_sourcing_epic_id methods: notes: - :type @@ -357,6 +382,7 @@ ee: - design_versions: - actions: - :design # Duplicate export of issues.designs in order to link the record to both Issue and Action + - :epic - protected_branches: - :unprotect_access_levels - protected_environments: diff --git a/lib/gitlab/import_export/import_failure_service.rb b/lib/gitlab/import_export/import_failure_service.rb new file mode 100644 index 00000000000..eeaf10870c8 --- /dev/null +++ b/lib/gitlab/import_export/import_failure_service.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class ImportFailureService + RETRIABLE_EXCEPTIONS = [GRPC::DeadlineExceeded, ActiveRecord::QueryCanceled].freeze + + attr_reader :importable + + def initialize(importable) + @importable = importable + @association = importable.association(:import_failures) + end + + def with_retry(relation_key, relation_index) + on_retry = -> (exception, retry_count, *_args) do + log_import_failure(relation_key, relation_index, exception, retry_count) + end + + Retriable.with_context(:relation_import, on_retry: on_retry) do + yield + end + end + + def log_import_failure(relation_key, relation_index, exception, retry_count = 0) + extra = { + relation_key: relation_key, + relation_index: relation_index, + retry_count: retry_count + } + extra[importable_column_name] = importable.id + + Gitlab::ErrorTracking.track_exception(exception, extra) + + attributes = { + exception_class: exception.class.to_s, + exception_message: exception.message.truncate(255), + correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id + }.merge(extra) + + ImportFailure.create(attributes) + end + + private + + def importable_column_name + @importable_column_name ||= @association.reflection.foreign_key.to_sym + end + end + end +end diff --git a/lib/gitlab/import_export/project_relation_factory.rb b/lib/gitlab/import_export/project_relation_factory.rb new file mode 100644 index 00000000000..e27bb9d3af1 --- /dev/null +++ b/lib/gitlab/import_export/project_relation_factory.rb @@ -0,0 +1,184 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class ProjectRelationFactory < BaseRelationFactory + prepend_if_ee('::EE::Gitlab::ImportExport::ProjectRelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule + + OVERRIDES = { snippets: :project_snippets, + ci_pipelines: 'Ci::Pipeline', + pipelines: 'Ci::Pipeline', + stages: 'Ci::Stage', + statuses: 'commit_status', + triggers: 'Ci::Trigger', + pipeline_schedules: 'Ci::PipelineSchedule', + builds: 'Ci::Build', + runners: 'Ci::Runner', + hooks: 'ProjectHook', + merge_access_levels: 'ProtectedBranch::MergeAccessLevel', + push_access_levels: 'ProtectedBranch::PushAccessLevel', + create_access_levels: 'ProtectedTag::CreateAccessLevel', + labels: :project_labels, + priorities: :label_priorities, + auto_devops: :project_auto_devops, + label: :project_label, + custom_attributes: 'ProjectCustomAttribute', + project_badges: 'Badge', + metrics: 'MergeRequest::Metrics', + ci_cd_settings: 'ProjectCiCdSetting', + error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', + links: 'Releases::Link', + metrics_setting: 'ProjectMetricsSetting' }.freeze + + BUILD_MODELS = %i[Ci::Build commit_status].freeze + + GROUP_REFERENCES = %w[group_id].freeze + + PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze + + EXISTING_OBJECT_RELATIONS = %i[ + milestone + milestones + label + labels + project_label + project_labels + group_label + group_labels + project_feature + merge_request + epic + ProjectCiCdSetting + container_expiration_policy + ].freeze + + def create + @object = super + + # We preload the project, user, and group to re-use objects + @object = preload_keys(@object, PROJECT_REFERENCES, @importable) + @object = preload_keys(@object, GROUP_REFERENCES, @importable.group) + @object = preload_keys(@object, USER_REFERENCES, @user) + end + + private + + def invalid_relation? + # Do not create relation if it is: + # - An unknown service + # - A legacy trigger + unknown_service? || + (!Feature.enabled?(:use_legacy_pipeline_triggers, @importable) && legacy_trigger?) + end + + def setup_models + case @relation_name + when :merge_request_diff_files then setup_diff + when :notes then setup_note + when :'Ci::Pipeline' then setup_pipeline + when *BUILD_MODELS then setup_build + end + + update_project_references + update_group_references + end + + def generate_imported_object + if @relation_name == :merge_requests + MergeRequestParser.new(@importable, @relation_hash.delete('diff_head_sha'), super, @relation_hash).parse! + else + super + end + end + + def update_project_references + # If source and target are the same, populate them with the new project ID. + if @relation_hash['source_project_id'] + @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID + end + + @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id'] + end + + def same_source_and_target? + @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] + end + + def update_group_references + return unless existing_object? + return unless @relation_hash['group_id'] + + @relation_hash['group_id'] = @importable.namespace_id + end + + # This code is a workaround for broken project exports that don't + # export merge requests with CI pipelines (i.e. exports that were + # generated from + # https://gitlab.com/gitlab-org/gitlab/merge_requests/17844). + # This method can be removed in GitLab 12.6. + def update_merge_request_references + # If a merge request was properly created, we don't need to fix + # up this export. + return if @relation_hash['merge_request'] + + merge_request_id = @relation_hash['merge_request_id'] + + return unless merge_request_id + + new_merge_request_id = @merge_requests_mapping[merge_request_id] + + return unless new_merge_request_id + + @relation_hash['merge_request_id'] = new_merge_request_id + parsed_relation_hash['merge_request_id'] = new_merge_request_id + end + + def setup_build + @relation_hash.delete('trace') # old export files have trace + @relation_hash.delete('token') + @relation_hash.delete('commands') + @relation_hash.delete('artifacts_file_store') + @relation_hash.delete('artifacts_metadata_store') + @relation_hash.delete('artifacts_size') + end + + def setup_diff + @relation_hash['diff'] = @relation_hash.delete('utf8_diff') + end + + def setup_pipeline + update_merge_request_references + + @relation_hash.fetch('stages', []).each do |stage| + stage.statuses.each do |status| + status.pipeline = imported_object + end + end + end + + def unknown_service? + @relation_name == :services && parsed_relation_hash['type'] && + !Object.const_defined?(parsed_relation_hash['type']) + end + + def legacy_trigger? + @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil? + end + + def preload_keys(object, references, value) + return object unless value + + references.each do |key| + attribute = "#{key.delete_suffix('_id')}=".to_sym + next unless object.respond_to?(key) && object.respond_to?(attribute) + + if object.read_attribute(key) == value&.id + object.public_send(attribute, value) # rubocop:disable GitlabSecurity/PublicSend + end + end + + object + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index e274b68a94f..e598cfc143e 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -10,7 +10,7 @@ module Gitlab def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @user = user - @shared = shared + @shared = shared @project = project end @@ -48,6 +48,7 @@ module Gitlab shared: @shared, importable: @project, tree_hash: @tree_hash, + object_builder: object_builder, members_mapper: members_mapper, relation_factory: relation_factory, reader: reader @@ -60,8 +61,12 @@ module Gitlab importable: @project) end + def object_builder + Gitlab::ImportExport::GroupProjectObjectBuilder + end + def relation_factory - Gitlab::ImportExport::RelationFactory + Gitlab::ImportExport::ProjectRelationFactory end def reader diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index d9c253788b4..44cf90fb86a 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -4,19 +4,20 @@ module Gitlab module ImportExport class RelationTreeRestorer # Relations which cannot be saved at project level (and have a group assigned) - GROUP_MODELS = [GroupLabel, Milestone].freeze + GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze attr_reader :user attr_reader :shared attr_reader :importable attr_reader :tree_hash - def initialize(user:, shared:, importable:, tree_hash:, members_mapper:, relation_factory:, reader:) + def initialize(user:, shared:, importable:, tree_hash:, members_mapper:, object_builder:, relation_factory:, reader:) @user = user @shared = shared @importable = importable @tree_hash = tree_hash @members_mapper = members_mapper + @object_builder = object_builder @relation_factory = relation_factory @reader = reader end @@ -71,28 +72,18 @@ module Gitlab return if importable_class == Project && group_model?(relation_object) relation_object.assign_attributes(importable_class_sym => @importable) - relation_object.save! + + import_failure_service.with_retry(relation_key, relation_index) do + relation_object.save! + end save_id_mapping(relation_key, data_hash, relation_object) rescue => e - # re-raise if not enabled - raise e unless Feature.enabled?(:import_graceful_failures, @importable.group, default_enabled: true) - - log_import_failure(relation_key, relation_index, e) + import_failure_service.log_import_failure(relation_key, relation_index, e) end - def log_import_failure(relation_key, relation_index, exception) - Gitlab::ErrorTracking.track_exception(exception, - project_id: @importable.id, relation_key: relation_key, relation_index: relation_index) - - ImportFailure.create( - project: @importable, - relation_key: relation_key, - relation_index: relation_index, - exception_class: exception.class.to_s, - exception_message: exception.message.truncate(255), - correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id - ) + def import_failure_service + @import_failure_service ||= ImportFailureService.new(@importable) end # Older, serialized CI pipeline exports may only have a @@ -224,15 +215,16 @@ module Gitlab def relation_factory_params(relation_key, data_hash) base_params = { - relation_sym: relation_key.to_sym, - relation_hash: data_hash, + relation_sym: relation_key.to_sym, + relation_hash: data_hash, + importable: @importable, members_mapper: @members_mapper, - user: @user, - excluded_keys: excluded_keys_for_relation(relation_key) + object_builder: @object_builder, + user: @user, + excluded_keys: excluded_keys_for_relation(relation_key) } base_params[:merge_requests_mapping] = merge_requests_mapping if importable_class == Project - base_params[importable_class_sym] = @importable base_params end end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index 8230c0f1e77..dab8bbf539d 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -13,6 +13,8 @@ module Gitlab mkdir_p(@shared.export_path) File.write(version_file, Gitlab::ImportExport.version, mode: 'w') + File.write(gitlab_version_file, Gitlab::VERSION, mode: 'w') + File.write(gitlab_revision_file, Gitlab.revision, mode: 'w') rescue => e @shared.error(e) false @@ -20,6 +22,14 @@ module Gitlab private + def gitlab_version_file + File.join(@shared.export_path, Gitlab::ImportExport.gitlab_version_filename) + end + + def gitlab_revision_file + File.join(@shared.export_path, Gitlab::ImportExport.gitlab_revision_filename) + end + def version_file File.join(@shared.export_path, Gitlab::ImportExport.version_filename) end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index b5181670b93..c7c348ce9eb 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -6,6 +6,7 @@ module Gitlab HELM_VERSION = '2.16.1' KUBECTL_VERSION = '1.13.12' NAMESPACE = 'gitlab-managed-apps' + NAMESPACE_LABELS = { 'app.gitlab.com/managed_by' => :gitlab }.freeze SERVICE_ACCOUNT = 'tiller' CLUSTER_ROLE_BINDING = 'tiller-admin' CLUSTER_ROLE = 'cluster-admin' diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index 978cafae9ac..3ed07818302 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -6,7 +6,11 @@ module Gitlab class Api def initialize(kubeclient) @kubeclient = kubeclient - @namespace = Gitlab::Kubernetes::Namespace.new(Gitlab::Kubernetes::Helm::NAMESPACE, kubeclient) + @namespace = Gitlab::Kubernetes::Namespace.new( + Gitlab::Kubernetes::Helm::NAMESPACE, + kubeclient, + labels: Gitlab::Kubernetes::Helm::NAMESPACE_LABELS + ) end def install(command) diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 66c28a9b702..7cb7f46a623 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -17,6 +17,7 @@ module Gitlab core: { group: 'api', version: 'v1' }, rbac: { group: 'apis/rbac.authorization.k8s.io', version: 'v1' }, extensions: { group: 'apis/extensions', version: 'v1beta1' }, + istio: { group: 'apis/networking.istio.io', version: 'v1alpha3' }, knative: { group: 'apis/serving.knative.dev', version: 'v1alpha1' } }.freeze @@ -83,6 +84,13 @@ module Gitlab :watch_pod_log, to: :core_client + # Gateway methods delegate to the apis/networking.istio.io api + # group client + delegate :create_gateway, + :get_gateway, + :update_gateway, + to: :istio_client + attr_reader :api_prefix, :kubeclient_options # We disable redirects through 'http_max_redirects: 0', diff --git a/lib/gitlab/kubernetes/namespace.rb b/lib/gitlab/kubernetes/namespace.rb index 8a3bea95a04..9862861118b 100644 --- a/lib/gitlab/kubernetes/namespace.rb +++ b/lib/gitlab/kubernetes/namespace.rb @@ -3,11 +3,12 @@ module Gitlab module Kubernetes class Namespace - attr_accessor :name + attr_accessor :name, :labels - def initialize(name, client) + def initialize(name, client, labels: nil) @name = name @client = client + @labels = labels end def exists? @@ -17,7 +18,7 @@ module Gitlab end def create! - resource = ::Kubeclient::Resource.new(metadata: { name: name }) + resource = ::Kubeclient::Resource.new(metadata: { name: name, labels: labels }) log_event(:begin_create) @client.create_namespace(resource) diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index b23efd64dee..34634d20a16 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -80,7 +80,7 @@ module Gitlab if host.present? && api_version.present? "#{host}/api/#{api_version}" else - github_options[:site] + github_options[:site] || ::Octokit::Default.api_endpoint end end diff --git a/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb index 4f5e9a98799..e085f551952 100644 --- a/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb @@ -16,7 +16,7 @@ module Gitlab private def endpoint_for_metric(metric) - if ENV['USE_SAMPLE_METRICS'] + if params[:sample_metrics] Gitlab::Routing.url_helpers.sample_metrics_project_environment_path( project, params[:environment], diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index 269d90fa971..1f252572461 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -150,7 +150,7 @@ module Gitlab # Returns the prefix to use for the name of a series. def series_prefix - @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' + @series_prefix ||= Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_' end # Allow access from other metrics related middlewares diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index 1eae0a7bf45..4e16e335bee 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -39,14 +39,10 @@ module Gitlab end def add_metric(series, values, tags = {}) - prefix = sidekiq? ? 'sidekiq_' : 'rails_' + prefix = Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_' @metrics << Metric.new("#{prefix}#{series}", values, tags) end - - def sidekiq? - Sidekiq.server? - end end end end diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index 355f938704e..8c4d150adad 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -61,7 +61,7 @@ module Gitlab # it takes around 80ms. The instances of HttpServers are not a subject # to change so we can cache the list of servers. def http_servers - return [] unless defined?(::Unicorn::HttpServer) + return [] unless Gitlab::Runtime.unicorn? @http_servers ||= ObjectSpace.each_object(::Unicorn::HttpServer).to_a end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 2ed5878286a..5bd21b8e5d1 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -36,7 +36,7 @@ module Gitlab end def relative_path(path) - path.gsub(%r{^#{Rails.root.to_s}/?}, '') + path.gsub(%r{^#{Rails.root}/?}, '') end def values_for(event) diff --git a/lib/gitlab/middleware/correlation_id.rb b/lib/gitlab/middleware/correlation_id.rb deleted file mode 100644 index fffd5da827f..00000000000 --- a/lib/gitlab/middleware/correlation_id.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -# A dumb middleware that steals correlation id -# and sets it as a global context for the request -module Gitlab - module Middleware - class CorrelationId - include ActionView::Helpers::TagHelper - - def initialize(app) - @app = app - end - - def call(env) - ::Labkit::Correlation::CorrelationId.use_id(correlation_id(env)) do - @app.call(env) - end - end - - private - - def correlation_id(env) - request(env).request_id - end - - def request(env) - ActionDispatch::Request.new(env) - end - end - end -end diff --git a/lib/gitlab/middleware/request_context.rb b/lib/gitlab/middleware/request_context.rb new file mode 100644 index 00000000000..953423b371c --- /dev/null +++ b/lib/gitlab/middleware/request_context.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Middleware + class RequestContext + def initialize(app) + @app = app + end + + def call(env) + # We should be using ActionDispatch::Request instead of + # Rack::Request to be consistent with Rails, but due to a Rails + # bug described in + # https://gitlab.com/gitlab-org/gitlab-foss/issues/58573#note_149799010 + # hosts behind a load balancer will only see 127.0.0.1 for the + # load balancer's IP. + req = Rack::Request.new(env) + + Gitlab::RequestContext.instance.client_ip = req.ip + Gitlab::RequestContext.instance.start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + Gitlab::RequestContext.instance.request_start_time = Gitlab::Metrics::System.real_time + + @app.call(env) + end + end + end +end diff --git a/lib/gitlab/multi_destination_logger.rb b/lib/gitlab/multi_destination_logger.rb new file mode 100644 index 00000000000..b6b19e81389 --- /dev/null +++ b/lib/gitlab/multi_destination_logger.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + class MultiDestinationLogger < ::Logger + def close + loggers.each(&:close) + end + + def self.debug(message) + loggers.each { |logger| logger.build.debug(message) } + end + + def self.error(message) + loggers.each { |logger| logger.build.error(message) } + end + + def self.warn(message) + loggers.each { |logger| logger.build.warn(message) } + end + + def self.info(message) + loggers.each { |logger| logger.build.info(message) } + end + + def self.read_latest + primary_logger.read_latest + end + + def self.file_name + primary_logger.file_name + end + + def self.full_log_path + primary_logger.full_log_path + end + + def self.file_name_noext + primary_logger.file_name_noext + end + + def self.loggers + raise NotImplementedError + end + + def self.primary_logger + raise NotImplementedError + end + end +end diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb index 4899b1d3234..c8cb8b6e020 100644 --- a/lib/gitlab/pages.rb +++ b/lib/gitlab/pages.rb @@ -4,6 +4,7 @@ module Gitlab class Pages VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze INTERNAL_API_REQUEST_HEADER = 'Gitlab-Pages-Api-Request'.freeze + MAX_SIZE = 1.terabyte include JwtAuthenticatable @@ -17,6 +18,11 @@ module Gitlab def secret_path Gitlab.config.pages.secret_file end + + def access_control_is_forced? + ::Gitlab.config.pages.access_control && + ::Gitlab::CurrentSettings.current_application_settings.force_pages_access_control + end end end end diff --git a/lib/gitlab/pagination/base.rb b/lib/gitlab/pagination/base.rb index 90fa1f8d1ec..a8a3397eba2 100644 --- a/lib/gitlab/pagination/base.rb +++ b/lib/gitlab/pagination/base.rb @@ -3,29 +3,12 @@ module Gitlab module Pagination class Base - private - - def per_page - @per_page ||= params[:per_page] - end - - def base_request_uri - @base_request_uri ||= URI.parse(request.url).tap do |uri| - uri.host = Gitlab.config.gitlab.host - uri.port = Gitlab.config.gitlab.port - end + def paginate(relation) + raise NotImplementedError end - def build_page_url(query_params:) - base_request_uri.tap do |uri| - uri.query = query_params - end.to_s - end - - def page_href(next_page_params = {}) - query_params = params.merge(**next_page_params, per_page: per_page).to_query - - build_page_url(query_params: query_params) + def finalize(records) + # Optional: Called with the actual set of records end end end diff --git a/lib/gitlab/pagination/keyset.rb b/lib/gitlab/pagination/keyset.rb index 5bd45fa9b56..8692f30e165 100644 --- a/lib/gitlab/pagination/keyset.rb +++ b/lib/gitlab/pagination/keyset.rb @@ -3,10 +3,6 @@ module Gitlab module Pagination module Keyset - def self.paginate(request_context, relation) - Gitlab::Pagination::Keyset::Pager.new(request_context).paginate(relation) - end - def self.available?(request_context, relation) order_by = request_context.page.order_by diff --git a/lib/gitlab/pagination/keyset/page.rb b/lib/gitlab/pagination/keyset/page.rb index 735f54faf0f..8070512f973 100644 --- a/lib/gitlab/pagination/keyset/page.rb +++ b/lib/gitlab/pagination/keyset/page.rb @@ -11,14 +11,13 @@ module Gitlab # Maximum number of records for a page MAXIMUM_PAGE_SIZE = 100 - attr_accessor :lower_bounds, :end_reached + attr_accessor :lower_bounds attr_reader :order_by - def initialize(order_by: {}, lower_bounds: nil, per_page: DEFAULT_PAGE_SIZE, end_reached: false) + def initialize(order_by: {}, lower_bounds: nil, per_page: DEFAULT_PAGE_SIZE) @order_by = order_by.symbolize_keys @lower_bounds = lower_bounds&.symbolize_keys @per_page = per_page - @end_reached = end_reached end # Number of records to return per page @@ -28,17 +27,11 @@ module Gitlab [@per_page, MAXIMUM_PAGE_SIZE].min end - # Determine whether this page indicates the end of the collection - def end_reached? - @end_reached - end - # Construct a Page for the next page # Uses identical order_by/per_page information for the next page - def next(lower_bounds, end_reached) + def next(lower_bounds) dup.tap do |next_page| next_page.lower_bounds = lower_bounds&.symbolize_keys - next_page.end_reached = end_reached end end end diff --git a/lib/gitlab/pagination/keyset/pager.rb b/lib/gitlab/pagination/keyset/pager.rb index 99b125cc2a0..6a2ae20f3b8 100644 --- a/lib/gitlab/pagination/keyset/pager.rb +++ b/lib/gitlab/pagination/keyset/pager.rb @@ -3,7 +3,7 @@ module Gitlab module Pagination module Keyset - class Pager + class Pager < Gitlab::Pagination::Base attr_reader :request def initialize(request) @@ -14,27 +14,20 @@ module Gitlab # Validate assumption: The last two columns must match the page order_by validate_order!(relation) - # This performs the database query and retrieves records - # We retrieve one record more to check if we have data beyond this page - all_records = relation.limit(page.per_page + 1).to_a # rubocop: disable CodeReuse/ActiveRecord - - records_for_page = all_records.first(page.per_page) - - # If we retrieved more records than belong on this page, - # we know there's a next page - there_is_more = all_records.size > records_for_page.size - apply_headers(records_for_page.last, there_is_more) + relation.limit(page.per_page) # rubocop: disable CodeReuse/ActiveRecord + end - records_for_page + def finalize(records) + apply_headers(records.last) end private - def apply_headers(last_record_in_page, there_is_more) - end_reached = last_record_in_page.nil? || !there_is_more - lower_bounds = last_record_in_page&.slice(page.order_by.keys) + def apply_headers(last_record_in_page) + return unless last_record_in_page - next_page = page.next(lower_bounds, end_reached) + lower_bounds = last_record_in_page&.slice(page.order_by.keys) + next_page = page.next(lower_bounds) request.apply_headers(next_page) end diff --git a/lib/gitlab/pagination/keyset/request_context.rb b/lib/gitlab/pagination/keyset/request_context.rb index aeaed7587b3..8c8138b3076 100644 --- a/lib/gitlab/pagination/keyset/request_context.rb +++ b/lib/gitlab/pagination/keyset/request_context.rb @@ -68,8 +68,6 @@ module Gitlab end def pagination_links(next_page) - return if next_page.end_reached? - %(<#{page_href(next_page)}>; rel="next") end diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index bf31f252a6b..11a5ef4e518 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -72,6 +72,29 @@ module Gitlab def data_without_counts?(paginated_data) paginated_data.is_a?(Kaminari::PaginatableWithoutCount) end + + def base_request_uri + @base_request_uri ||= URI.parse(request.url).tap do |uri| + uri.host = Gitlab.config.gitlab.host + uri.port = Gitlab.config.gitlab.port + end + end + + def build_page_url(query_params:) + base_request_uri.tap do |uri| + uri.query = query_params + end.to_s + end + + def page_href(next_page_params = {}) + query_params = params.merge(**next_page_params, per_page: per_page).to_query + + build_page_url(query_params: query_params) + end + + def per_page + @per_page ||= params[:per_page] + end end end end diff --git a/lib/gitlab/patch/action_dispatch_journey_formatter.rb b/lib/gitlab/patch/action_dispatch_journey_formatter.rb new file mode 100644 index 00000000000..2d3b7bb9923 --- /dev/null +++ b/lib/gitlab/patch/action_dispatch_journey_formatter.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Patch + module ActionDispatchJourneyFormatter + def self.prepended(mod) + mod.alias_method(:old_missing_keys, :missing_keys) + mod.remove_method(:missing_keys) + end + + private + + def missing_keys(route, parts) + missing_keys = nil + tests = route.path.requirements_for_missing_keys_check + route.required_parts.each do |key| + case tests[key] + when nil + unless parts[key] + missing_keys ||= [] + missing_keys << key + end + else + unless tests[key].match?(parts[key]) + missing_keys ||= [] + missing_keys << key + end + end + end + missing_keys + end + end + end +end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index f2f6180c464..f47ccb8fed9 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -107,7 +107,7 @@ module Gitlab super - Gitlab::Profiler.clean_backtrace(caller).each do |caller_line| + Gitlab::BacktraceCleaner.clean_backtrace(caller).each do |caller_line| stripped_caller_line = caller_line.sub("#{Rails.root}/", '') super(" ↳ #{stripped_caller_line}") @@ -117,14 +117,6 @@ module Gitlab end end - def self.clean_backtrace(backtrace) - return unless backtrace - - Array(Rails.backtrace_cleaner.clean(backtrace)).reject do |line| - line.match(Regexp.union(IGNORE_BACKTRACES)) - end - end - def self.with_custom_logger(logger) original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging original_activerecord_logger = ActiveRecord::Base.logger diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb index 4e5e2d4a6a9..e2271b1492c 100644 --- a/lib/gitlab/project_authorizations.rb +++ b/lib/gitlab/project_authorizations.rb @@ -68,7 +68,7 @@ module Gitlab .select([namespaces[:id], members[:access_level]]) .except(:order) - if Feature.enabled?(:share_group_with_group) + if Feature.enabled?(:share_group_with_group, default_enabled: true) # Namespaces shared with any of the group cte << Group.select([namespaces[:id], 'group_group_links.group_access AS access_level']) .joins(join_group_group_links) diff --git a/lib/gitlab/prometheus/adapter.rb b/lib/gitlab/prometheus/adapter.rb new file mode 100644 index 00000000000..ed10ef2917f --- /dev/null +++ b/lib/gitlab/prometheus/adapter.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Prometheus + class Adapter + attr_reader :project, :cluster + + def initialize(project, cluster) + @project = project + @cluster = cluster + end + + def prometheus_adapter + @prometheus_adapter ||= if service_prometheus_adapter.can_query? + service_prometheus_adapter + else + cluster_prometheus_adapter + end + end + + def cluster_prometheus_adapter + application = cluster&.application_prometheus + + application if application&.available? + end + + private + + def service_prometheus_adapter + project.find_or_initialize_service('prometheus') + end + end + end +end diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 333f848df9b..02446a7953b 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -32,6 +32,8 @@ module Gitlab OPTION_MATCHER = /(?<namespace>[^\.]+)\.(?<key>[^=]+)=?(?<value>.*)/.freeze + CI_SKIP = 'ci.skip' + attr_reader :options def initialize(options = []) diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index ebdae139315..b17a0208f95 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -4,7 +4,7 @@ module Gitlab module QuickActions class CommandDefinition attr_accessor :name, :aliases, :description, :explanation, :execution_message, - :params, :condition_block, :parse_params_block, :action_block, :warning, :types + :params, :condition_block, :parse_params_block, :action_block, :warning, :icon, :types def initialize(name, attributes = {}) @name = name @@ -12,6 +12,7 @@ module Gitlab @aliases = attributes[:aliases] || [] @description = attributes[:description] || '' @warning = attributes[:warning] || '' + @icon = attributes[:icon] || '' @explanation = attributes[:explanation] || '' @execution_message = attributes[:execution_message] || '' @params = attributes[:params] || [] @@ -45,7 +46,13 @@ module Gitlab explanation end - warning.empty? ? message : "#{message} (#{warning})" + warning_text = if warning.respond_to?(:call) + execute_block(warning, context, arg) + else + warning + end + + warning.empty? ? message : "#{message} (#{warning_text})" end def execute(context, arg) @@ -72,6 +79,11 @@ module Gitlab desc = context.instance_exec(&desc) rescue '' end + warn = warning + if warn.respond_to?(:call) + warn = context.instance_exec(&warn) rescue '' + end + prms = params if prms.respond_to?(:call) prms = Array(context.instance_exec(&prms)) rescue params @@ -81,7 +93,8 @@ module Gitlab name: name, aliases: aliases, description: desc, - warning: warning, + warning: warn, + icon: icon, params: prms } end diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index 5abbd377642..a2dfcc6de9a 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -33,8 +33,12 @@ module Gitlab @description = block_given? ? block : text end - def warning(message = '') - @warning = message + def warning(text = '', &block) + @warning = block_given? ? block : text + end + + def icon(string = '') + @icon = string end # Allows to define params for the next quick action. @@ -192,6 +196,7 @@ module Gitlab aliases: aliases, description: @description, warning: @warning, + icon: @icon, explanation: @explanation, execution_message: @execution_message, params: @params, @@ -213,6 +218,7 @@ module Gitlab @params = nil @condition_block = nil @warning = nil + @icon = nil @parse_params_block = nil @types = nil end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 412d00c6939..c8932b26925 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -22,11 +22,8 @@ module Gitlab def pool_size # heuristic constant 5 should be a config setting somewhere -- related to CPU count? size = 5 - if Sidekiq.server? - # the pool will be used in a multi-threaded context - size += Sidekiq.options[:concurrency] - elsif defined?(::Puma) - size += Puma.cli_config.options[:max_threads] + if Gitlab::Runtime.multi_threaded? + size += Gitlab::Runtime.max_threads end size diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index d9300da38a5..48eaf073e8a 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -5,7 +5,7 @@ module Gitlab extend self def project_name_regex - @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9c0}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9c0}_\. ]*\z/.freeze + @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9ff}_\. ]*\z/.freeze end def project_name_regex_message diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 030e50dfbf6..1baa2a9e461 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -32,9 +32,12 @@ module Gitlab def self.find_project(project_path) project = Project.find_by_full_path(project_path, follow_redirects: true) - was_redirected = project && project.full_path.casecmp(project_path) != 0 - [project, was_redirected] + [project, redirected?(project, project_path)] + end + + def self.redirected?(project, project_path) + project && project.full_path.casecmp(project_path) != 0 end end end diff --git a/lib/gitlab/repository_cache.rb b/lib/gitlab/repository_cache.rb index 56007574b1b..fca8c43da2e 100644 --- a/lib/gitlab/repository_cache.rb +++ b/lib/gitlab/repository_cache.rb @@ -7,7 +7,8 @@ module Gitlab def initialize(repository, extra_namespace: nil, backend: Rails.cache) @repository = repository - @namespace = "#{repository.full_path}:#{repository.project.id}" + @namespace = "#{repository.full_path}" + @namespace += ":#{repository.project.id}" if repository.project @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace @backend = backend end diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index 6d3ac53a787..4797ec0b116 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -7,7 +7,8 @@ module Gitlab def initialize(repository, extra_namespace: nil, expires_in: 2.weeks) @repository = repository - @namespace = "#{repository.full_path}:#{repository.project.id}" + @namespace = "#{repository.full_path}" + @namespace += ":#{repository.project.id}" if repository.project @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace @expires_in = expires_in end diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb index 13187836e02..214670cac28 100644 --- a/lib/gitlab/request_context.rb +++ b/lib/gitlab/request_context.rb @@ -2,34 +2,40 @@ module Gitlab class RequestContext - class << self - def client_ip - Gitlab::SafeRequestStore[:client_ip] - end + include Gitlab::Utils::StrongMemoize + include Singleton + + RequestDeadlineExceeded = Class.new(StandardError) + + attr_accessor :client_ip, :start_thread_cpu_time, :request_start_time - def start_thread_cpu_time - Gitlab::SafeRequestStore[:start_thread_cpu_time] + class << self + def instance + Gitlab::SafeRequestStore[:request_context] ||= new end end - def initialize(app) - @app = app + def request_deadline + strong_memoize(:request_deadline) do + next unless request_start_time + next unless Feature.enabled?(:request_deadline) + + request_start_time + max_request_duration_seconds + end end - def call(env) - # We should be using ActionDispatch::Request instead of - # Rack::Request to be consistent with Rails, but due to a Rails - # bug described in - # https://gitlab.com/gitlab-org/gitlab-foss/issues/58573#note_149799010 - # hosts behind a load balancer will only see 127.0.0.1 for the - # load balancer's IP. - req = Rack::Request.new(env) + def ensure_deadline_not_exceeded! + return unless request_deadline + return if Gitlab::Metrics::System.real_time < request_deadline - Gitlab::SafeRequestStore[:client_ip] = req.ip + raise RequestDeadlineExceeded, + "Request takes longer than #{max_request_duration_seconds}" + end - Gitlab::SafeRequestStore[:start_thread_cpu_time] = Gitlab::Metrics::System.thread_cpu_time + private - @app.call(env) + def max_request_duration_seconds + Settings.gitlab.max_request_duration_seconds end end end diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb new file mode 100644 index 00000000000..97f7a8e2800 --- /dev/null +++ b/lib/gitlab/runtime.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + # Provides routines to identify the current runtime as which the application + # executes, such as whether it is an application server and which one. + module Runtime + IdentificationError = Class.new(RuntimeError) + AmbiguousProcessError = Class.new(IdentificationError) + UnknownProcessError = Class.new(IdentificationError) + + class << self + def identify + matches = [] + matches << :puma if puma? + matches << :unicorn if unicorn? + matches << :console if console? + matches << :sidekiq if sidekiq? + matches << :rake if rake? + matches << :rspec if rspec? + + if matches.one? + matches.first + elsif matches.none? + raise UnknownProcessError.new( + "Failed to identify runtime for process #{Process.pid} (#{$0})" + ) + else + raise AmbiguousProcessError.new( + "Ambiguous runtime #{matches} for process #{Process.pid} (#{$0})" + ) + end + end + + def puma? + !!defined?(::Puma) + end + + # For unicorn, we need to check for actual server instances to avoid false positives. + def unicorn? + !!(defined?(::Unicorn) && defined?(::Unicorn::HttpServer)) + end + + def sidekiq? + !!(defined?(::Sidekiq) && Sidekiq.server?) + end + + def rake? + !!(defined?(::Rake) && Rake.application.top_level_tasks.any?) + end + + def rspec? + Rails.env.test? && process_name == 'rspec' + end + + def console? + !!defined?(::Rails::Console) + end + + def web_server? + puma? || unicorn? + end + + def multi_threaded? + puma? || sidekiq? + end + + def process_name + File.basename($0) + end + + def max_threads + if puma? + Puma.cli_config.options[:max_threads] + elsif sidekiq? + Sidekiq.options[:concurrency] + else + 1 + end + end + end + end +end diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb index 604b6df12cc..5d10d8c4877 100644 --- a/lib/gitlab/sherlock/file_sample.rb +++ b/lib/gitlab/sherlock/file_sample.rb @@ -18,7 +18,7 @@ module Gitlab end def relative_path - @relative_path ||= @file.gsub(%r{^#{Rails.root.to_s}/?}, '') + @relative_path ||= @file.gsub(%r{^#{Rails.root}/?}, '') end def to_param diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb index 209ba784f9c..52d88f074b7 100644 --- a/lib/gitlab/sherlock/line_profiler.rb +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -45,7 +45,7 @@ module Gitlab require 'rblineprof' retval = nil - samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield } + samples = lineprof(/^#{Rails.root}/) { retval = yield } file_samples = aggregate_rblineprof(samples) diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index ffceeb68f20..b246c507e9e 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -1,78 +1,53 @@ # frozen_string_literal: true require 'yaml' -require 'set' module Gitlab module SidekiqConfig - QUEUE_CONFIG_PATHS = begin - result = %w[app/workers/all_queues.yml] - result << 'ee/app/workers/all_queues.yml' if Gitlab.ee? - result - end.freeze + class << self + include Gitlab::SidekiqConfig::CliMethods - # This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside - # of bundler/Rails context, so we cannot use any gem or Rails methods. - def self.worker_queues(rails_path = Rails.root.to_s) - @worker_queues ||= {} - - @worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path| - full_path = File.join(rails_path, path) - - File.exist?(full_path) ? YAML.load_file(full_path) : [] + def redis_queues + # Not memoized, because this can change during the life of the application + Sidekiq::Queue.all.map(&:name) end - end - - # This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside - # of bundler/Rails context, so we cannot use any gem or Rails methods. - def self.expand_queues(queues, all_queues = self.worker_queues) - return [] if queues.empty? - queues_set = all_queues.to_set - - queues.flat_map do |queue| - [queue, *queues_set.grep(/\A#{queue}:/)] + def config_queues + @config_queues ||= begin + config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml')) + config[:queues].map(&:first) + end end - end - def self.redis_queues - # Not memoized, because this can change during the life of the application - Sidekiq::Queue.all.map(&:name) - end + def cron_workers + @cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize } + end - def self.config_queues - @config_queues ||= begin - config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml')) - config[:queues].map(&:first) + def workers + @workers ||= begin + result = find_workers(Rails.root.join('app', 'workers')) + result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'))) if Gitlab.ee? + result + end end - end - def self.cron_workers - @cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize } - end + private - def self.workers - @workers ||= begin - result = find_workers(Rails.root.join('app', 'workers')) - result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'))) if Gitlab.ee? - result - end - end + def find_workers(root) + concerns = root.join('concerns').to_s - def self.find_workers(root) - concerns = root.join('concerns').to_s + workers = Dir[root.join('**', '*.rb')] + .reject { |path| path.start_with?(concerns) } - workers = Dir[root.join('**', '*.rb')] - .reject { |path| path.start_with?(concerns) } + workers.map! do |path| + ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '') - workers.map! do |path| - ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '') + ns.camelize.constantize + end - ns.camelize.constantize + # Skip things that aren't workers + workers.select { |w| w < Sidekiq::Worker } end - - # Skip things that aren't workers - workers.select { |w| w < Sidekiq::Worker } end end end diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb new file mode 100644 index 00000000000..1ce46289e81 --- /dev/null +++ b/lib/gitlab/sidekiq_config/cli_methods.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'yaml' +require 'set' + +# These methods are called by `sidekiq-cluster`, which runs outside of +# the bundler/Rails context, so we cannot use any gem or Rails methods. +module Gitlab + module SidekiqConfig + module CliMethods + # The methods in this module are used as module methods + # rubocop:disable Gitlab/ModuleWithInstanceVariables + extend self + + QUEUE_CONFIG_PATHS = begin + result = %w[app/workers/all_queues.yml] + result << 'ee/app/workers/all_queues.yml' if Gitlab.ee? + result + end.freeze + + def worker_queues(rails_path = Rails.root.to_s) + @worker_queues ||= {} + + @worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path| + full_path = File.join(rails_path, path) + + File.exist?(full_path) ? YAML.load_file(full_path) : [] + end + end + + def expand_queues(queues, all_queues = self.worker_queues) + return [] if queues.empty? + + queues_set = all_queues.to_set + + queues.flat_map do |queue| + [queue, *queues_set.grep(/\A#{queue}:/)] + end + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + end + end +end diff --git a/lib/gitlab/sidekiq_logging/exception_handler.rb b/lib/gitlab/sidekiq_logging/exception_handler.rb index fba74b6c9ed..a6d6819bf8e 100644 --- a/lib/gitlab/sidekiq_logging/exception_handler.rb +++ b/lib/gitlab/sidekiq_logging/exception_handler.rb @@ -18,7 +18,7 @@ module Gitlab data.merge!(job_data) if job_data.present? end - data[:error_backtrace] = Gitlab::Profiler.clean_backtrace(job_exception.backtrace) if job_exception.backtrace.present? + data[:error_backtrace] = Gitlab::BacktraceCleaner.clean_backtrace(job_exception.backtrace) if job_exception.backtrace.present? Sidekiq.logger.warn(data) end diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb index 88888c5994e..e0b0d684bea 100644 --- a/lib/gitlab/sidekiq_logging/json_formatter.rb +++ b/lib/gitlab/sidekiq_logging/json_formatter.rb @@ -3,6 +3,8 @@ module Gitlab module SidekiqLogging class JSONFormatter + TIMESTAMP_FIELDS = %w[created_at enqueued_at started_at retried_at failed_at completed_at].freeze + def call(severity, timestamp, progname, data) output = { severity: severity, @@ -13,11 +15,27 @@ module Gitlab when String output[:message] = data when Hash + convert_to_iso8601!(data) output.merge!(data) end output.to_json + "\n" end + + private + + def convert_to_iso8601!(payload) + TIMESTAMP_FIELDS.each do |key| + value = payload[key] + payload[key] = format_time(value) if value.present? + end + end + + def format_time(timestamp) + return timestamp unless timestamp.is_a?(Numeric) + + Time.at(timestamp).utc.iso8601(3) + end end end end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index ca9e3b8428c..8e7626b8eb6 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -1,15 +1,17 @@ # frozen_string_literal: true +require 'active_record' +require 'active_record/log_subscriber' + module Gitlab module SidekiqLogging class StructuredLogger - START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze - DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze MAXIMUM_JOB_ARGUMENTS_LENGTH = 10.kilobytes def call(job, queue) started_time = get_time base_payload = parse_job(job) + ActiveRecord::LogSubscriber.reset_runtime Sidekiq.logger.info log_job_start(base_payload) @@ -61,7 +63,8 @@ module Gitlab payload['job_status'] = 'done' end - convert_to_iso8601(payload, DONE_TIMESTAMP_FIELDS) + payload['db_duration'] = ActiveRecord::LogSubscriber.runtime + payload['db_duration_s'] = payload['db_duration'] / 1000 payload end @@ -72,7 +75,7 @@ module Gitlab # ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0) # supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime payload['cpu_s'] = time[:cputime].round(6) if time[:cputime] > 0 - payload['completed_at'] = Time.now.utc + payload['completed_at'] = Time.now.utc.to_f end def parse_job(job) @@ -84,17 +87,9 @@ module Gitlab job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS'] job['args'] = limited_job_args(job['args']) if job['args'] - convert_to_iso8601(job, START_TIMESTAMP_FIELDS) - job end - def convert_to_iso8601(payload, keys) - keys.each do |key| - payload[key] = format_time(payload[key]) if payload[key] - end - end - def elapsed(t0) t1 = get_time { @@ -114,12 +109,6 @@ module Gitlab Gitlab::Metrics::System.monotonic_time end - def format_time(timestamp) - return timestamp if timestamp.is_a?(String) - - Time.at(timestamp).utc.iso8601(6) - end - def limited_job_args(args) return unless args.is_a?(Array) diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index c6726dcfa67..3dda244233f 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -10,12 +10,12 @@ module Gitlab def self.server_configurator(metrics: true, arguments_logger: true, memory_killer: true, request_store: true) lambda do |chain| chain.add Gitlab::SidekiqMiddleware::Monitor - chain.add Gitlab::SidekiqMiddleware::Metrics if metrics + chain.add Gitlab::SidekiqMiddleware::ServerMetrics if metrics chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger chain.add Gitlab::SidekiqMiddleware::MemoryKiller if memory_killer chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware if request_store chain.add Gitlab::SidekiqMiddleware::BatchLoader - chain.add Gitlab::SidekiqMiddleware::CorrelationLogger + chain.add Labkit::Middleware::Sidekiq::Server chain.add Gitlab::SidekiqMiddleware::InstrumentationLogger chain.add Gitlab::SidekiqStatus::ServerMiddleware end @@ -27,7 +27,8 @@ module Gitlab def self.client_configurator lambda do |chain| chain.add Gitlab::SidekiqStatus::ClientMiddleware - chain.add Gitlab::SidekiqMiddleware::CorrelationInjector + chain.add Gitlab::SidekiqMiddleware::ClientMetrics + chain.add Labkit::Middleware::Sidekiq::Client end end end diff --git a/lib/gitlab/sidekiq_middleware/client_metrics.rb b/lib/gitlab/sidekiq_middleware/client_metrics.rb new file mode 100644 index 00000000000..cd11415b55e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/client_metrics.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class ClientMetrics < SidekiqMiddleware::Metrics + ENQUEUED = :sidekiq_enqueued_jobs_total + + def initialize + @metrics = init_metrics + end + + def call(worker, _job, queue, _redis_pool) + labels = create_labels(worker.class, queue) + + @metrics.fetch(ENQUEUED).increment(labels, 1) + + yield + end + + private + + def init_metrics + { + ENQUEUED => ::Gitlab::Metrics.counter(ENQUEUED, 'Sidekiq jobs enqueued') + } + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/correlation_injector.rb b/lib/gitlab/sidekiq_middleware/correlation_injector.rb deleted file mode 100644 index 1539fd706ab..00000000000 --- a/lib/gitlab/sidekiq_middleware/correlation_injector.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module SidekiqMiddleware - class CorrelationInjector - def call(worker_class, job, queue, redis_pool) - job[Labkit::Correlation::CorrelationId::LOG_KEY] ||= - Labkit::Correlation::CorrelationId.current_or_new_id - - yield - end - end - end -end diff --git a/lib/gitlab/sidekiq_middleware/correlation_logger.rb b/lib/gitlab/sidekiq_middleware/correlation_logger.rb deleted file mode 100644 index cffc4483573..00000000000 --- a/lib/gitlab/sidekiq_middleware/correlation_logger.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module SidekiqMiddleware - class CorrelationLogger - def call(worker, job, queue) - correlation_id = job[Labkit::Correlation::CorrelationId::LOG_KEY] - - Labkit::Correlation::CorrelationId.use_id(correlation_id) do - yield - end - end - end - end -end diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb index 7bfb0d54d80..9588e9ef19a 100644 --- a/lib/gitlab/sidekiq_middleware/metrics.rb +++ b/lib/gitlab/sidekiq_middleware/metrics.rb @@ -3,68 +3,11 @@ module Gitlab module SidekiqMiddleware class Metrics - # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq - # timeframes than the DEFAULT_BUCKET definition. Defined in seconds. - SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze - TRUE_LABEL = "yes" FALSE_LABEL = "no" - def initialize - @metrics = init_metrics - - @metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) - end - - def call(worker, job, queue) - labels = create_labels(worker.class, queue) - queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) - - @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration - @metrics[:sidekiq_running_jobs].increment(labels, 1) - - if job['retry_count'].present? - @metrics[:sidekiq_jobs_retried_total].increment(labels, 1) - end - - job_succeeded = false - monotonic_time_start = Gitlab::Metrics::System.monotonic_time - job_thread_cputime_start = get_thread_cputime - begin - yield - job_succeeded = true - ensure - monotonic_time_end = Gitlab::Metrics::System.monotonic_time - job_thread_cputime_end = get_thread_cputime - - monotonic_time = monotonic_time_end - monotonic_time_start - job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start - - # sidekiq_running_jobs, sidekiq_jobs_failed_total should not include the job_status label - @metrics[:sidekiq_running_jobs].increment(labels, -1) - @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded - - # job_status: done, fail match the job_status attribute in structured logging - labels[:job_status] = job_succeeded ? "done" : "fail" - @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) - @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time) - end - end - private - def init_metrics - { - sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), - sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), - sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), - sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all) - } - end - def create_labels(worker_class, queue) labels = { queue: queue.to_s, latency_sensitive: FALSE_LABEL, external_dependencies: FALSE_LABEL, feature_category: "", boundary: "" } return labels unless worker_class.include? WorkerAttributes @@ -84,10 +27,6 @@ module Gitlab def bool_as_label(value) value ? TRUE_LABEL : FALSE_LABEL end - - def get_thread_cputime - defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 - end end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb new file mode 100644 index 00000000000..fa7f56b8d9c --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class ServerMetrics < SidekiqMiddleware::Metrics + # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq + # timeframes than the DEFAULT_BUCKET definition. Defined in seconds. + SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze + + def initialize + @metrics = init_metrics + + @metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) + end + + def call(worker, job, queue) + labels = create_labels(worker.class, queue) + queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) + + @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration + @metrics[:sidekiq_running_jobs].increment(labels, 1) + + if job['retry_count'].present? + @metrics[:sidekiq_jobs_retried_total].increment(labels, 1) + end + + job_succeeded = false + monotonic_time_start = Gitlab::Metrics::System.monotonic_time + job_thread_cputime_start = get_thread_cputime + begin + yield + job_succeeded = true + ensure + monotonic_time_end = Gitlab::Metrics::System.monotonic_time + job_thread_cputime_end = get_thread_cputime + + monotonic_time = monotonic_time_end - monotonic_time_start + job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start + + # sidekiq_running_jobs, sidekiq_jobs_failed_total should not include the job_status label + @metrics[:sidekiq_running_jobs].increment(labels, -1) + @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded + + # job_status: done, fail match the job_status attribute in structured logging + labels[:job_status] = job_succeeded ? "done" : "fail" + @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) + @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time) + end + end + + private + + def init_metrics + { + sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), + sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), + sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all) + } + end + + def get_thread_cputime + defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index ec2243345e1..e00b49b9042 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -178,18 +178,17 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def services_usage - types = { - SlackService: :projects_slack_notifications_active, - SlackSlashCommandsService: :projects_slack_slash_active, - PrometheusService: :projects_prometheus_active, - CustomIssueTrackerService: :projects_custom_issue_tracker_active, - JenkinsService: :projects_jenkins_active, - MattermostService: :projects_mattermost_active - } + service_counts = count(Service.active.where(template: false).where.not(type: 'JiraService').group(:type), fallback: Hash.new(-1)) + + results = Service.available_services_names.each_with_object({}) do |service_name, response| + response["projects_#{service_name}_active".to_sym] = service_counts["#{service_name}_service".camelize] || 0 + end + + # Keep old Slack keys for backward compatibility, https://gitlab.com/gitlab-data/analytics/issues/3241 + results[:projects_slack_notifications_active] = results[:projects_slack_active] + results[:projects_slack_slash_active] = results[:projects_slack_slash_commands_active] - results = count(Service.active.by_type(types.keys).group(:type), fallback: Hash.new(-1)) - types.each_with_object({}) { |(klass, key), response| response[key] = results[klass.to_s] || 0 } - .merge(jira_usage) + results.merge(jira_usage) end def jira_usage @@ -206,7 +205,6 @@ module Gitlab .by_type(:JiraService) .includes(:jira_tracker_data) .find_in_batches(batch_size: BATCH_SIZE) do |services| - counts = services.group_by do |service| # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 service_url = service.data_fields&.url || (service.properties && service.properties['url']) @@ -224,17 +222,17 @@ module Gitlab results end + # rubocop: enable CodeReuse/ActiveRecord def user_preferences_usage {} # augmented in EE end - def count(relation, count_by: nil, fallback: -1) - count_by ? relation.count(count_by) : relation.count + def count(relation, fallback: -1) + relation.count rescue ActiveRecord::StatementInvalid fallback end - # rubocop: enable CodeReuse/ActiveRecord def approximate_counts approx_counts = Gitlab::Database::Count.approximate_counts(APPROXIMATE_COUNT_MODELS) diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 7fbfc4c45c4..7eddfc471f6 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -50,6 +50,12 @@ module Gitlab .gsub(/(\A-+|-+\z)/, '') end + # Wraps ActiveSupport's Array#to_sentence to convert the given array to a + # comma-separated sentence joined with localized 'or' Strings instead of 'and'. + def to_exclusive_sentence(array) + array.to_sentence(two_words_connector: _(' or '), last_word_connector: _(', or ')) + end + # Converts newlines into HTML line break elements def nlbr(str) ActionView::Base.full_sanitizer.sanitize(+str, tags: []).gsub(/\r?\n/, '<br>').html_safe diff --git a/lib/gitlab/utils/lazy_attributes.rb b/lib/gitlab/utils/lazy_attributes.rb new file mode 100644 index 00000000000..79f3a7dcb53 --- /dev/null +++ b/lib/gitlab/utils/lazy_attributes.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + module LazyAttributes + extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize + + class_methods do + def lazy_attr_reader(*one_or_more_names, type: nil) + names = Array.wrap(one_or_more_names) + names.each { |name| define_lazy_reader(name, type: type) } + end + + def lazy_attr_accessor(*one_or_more_names, type: nil) + names = Array.wrap(one_or_more_names) + names.each do |name| + define_lazy_reader(name, type: type) + define_lazy_writer(name) + end + end + + private + + def define_lazy_reader(name, type:) + define_method(name) do + strong_memoize("#{name}_lazy_loaded") do + value = instance_variable_get("@#{name}") + value = value.call if value.respond_to?(:call) + value = nil if type && !value.is_a?(type) + value + end + end + end + + def define_lazy_writer(name) + define_method("#{name}=") do |value| + clear_memoization("#{name}_lazy_loaded") + instance_variable_set("@#{name}", value) + end + end + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 713ca31bbc5..29450a33289 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -22,18 +22,16 @@ module Gitlab def git_http_ok(repository, repo_type, user, action, show_all_refs: false) raise "Unsupported action: #{action}" unless ALLOWED_GIT_HTTP_ACTIONS.include?(action.to_s) - project = repository.project - attrs = { GL_ID: Gitlab::GlId.gl_id(user), - GL_REPOSITORY: repo_type.identifier_for_subject(project), + GL_REPOSITORY: repo_type.identifier_for_subject(repository.project), GL_USERNAME: user&.username, ShowAllRefs: show_all_refs, Repository: repository.gitaly_repository.to_h, GitConfigOptions: [], GitalyServer: { - address: Gitlab::GitalyClient.address(project.repository_storage), - token: Gitlab::GitalyClient.token(project.repository_storage), + address: Gitlab::GitalyClient.address(repository.storage), + token: Gitlab::GitalyClient.token(repository.storage), features: Feature::Gitaly.server_feature_flags } } diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb index 1bb3ddb964a..ed3470f81f4 100644 --- a/lib/peek/views/active_record.rb +++ b/lib/peek/views/active_record.rb @@ -32,7 +32,7 @@ module Peek detail_store << { duration: finish - start, sql: data[:sql].strip, - backtrace: Gitlab::Profiler.clean_backtrace(caller) + backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller) } end end diff --git a/lib/peek/views/redis_detailed.rb b/lib/peek/views/redis_detailed.rb index 84041b6be73..14cabd62025 100644 --- a/lib/peek/views/redis_detailed.rb +++ b/lib/peek/views/redis_detailed.rb @@ -23,7 +23,7 @@ module Gitlab detail_store << { cmd: args.first, duration: duration, - backtrace: ::Gitlab::Profiler.clean_backtrace(caller) + backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller) } end diff --git a/lib/prometheus/pid_provider.rb b/lib/prometheus/pid_provider.rb index 228639357ac..32beeb0d31e 100644 --- a/lib/prometheus/pid_provider.rb +++ b/lib/prometheus/pid_provider.rb @@ -5,11 +5,11 @@ module Prometheus extend self def worker_id - if Sidekiq.server? + if Gitlab::Runtime.sidekiq? sidekiq_worker_id - elsif defined?(Unicorn::Worker) + elsif Gitlab::Runtime.unicorn? unicorn_worker_id - elsif defined?(::Puma) + elsif Gitlab::Runtime.puma? puma_worker_id else unknown_process_id diff --git a/lib/sentry/api_urls.rb b/lib/sentry/api_urls.rb new file mode 100644 index 00000000000..388d0531da1 --- /dev/null +++ b/lib/sentry/api_urls.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Sentry + class ApiUrls + def initialize(url_base) + @uri = URI(url_base).freeze + end + + def issues_url + with_path(File.join(@uri.path, '/issues/')) + end + + def issue_url(issue_id) + with_path("/api/0/issues/#{escape(issue_id)}/") + end + + def projects_url + with_path('/api/0/projects/') + end + + def issue_latest_event_url(issue_id) + with_path("/api/0/issues/#{escape(issue_id)}/events/latest/") + end + + private + + def with_path(new_path) + new_uri = @uri.dup + # Sentry API returns 404 if there are extra slashes in the URL + new_uri.path = new_path.squeeze('/') + + new_uri + end + + def escape(param) + CGI.escape(param.to_s) + end + end +end diff --git a/lib/sentry/client.rb b/lib/sentry/client.rb index 3df688a1fda..8898960c24d 100644 --- a/lib/sentry/client.rb +++ b/lib/sentry/client.rb @@ -2,19 +2,15 @@ module Sentry class Client + include Sentry::Client::Event include Sentry::Client::Projects + include Sentry::Client::Issue + include Sentry::Client::Repo + include Sentry::Client::IssueLink Error = Class.new(StandardError) MissingKeysError = Class.new(StandardError) ResponseInvalidSizeError = Class.new(StandardError) - BadRequestError = Class.new(StandardError) - - SENTRY_API_SORT_VALUE_MAP = { - # <accepted_by_client> => <accepted_by_sentry_api> - 'frequency' => 'freq', - 'first_seen' => 'new', - 'last_seen' => nil - }.freeze attr_accessor :url, :token @@ -23,40 +19,10 @@ module Sentry @token = token end - def issue_details(issue_id:) - issue = get_issue(issue_id: issue_id) - - map_to_detailed_error(issue) - end - - def issue_latest_event(issue_id:) - latest_event = get_issue_latest_event(issue_id: issue_id) - - map_to_event(latest_event) - end - - def list_issues(**keyword_args) - response = get_issues(keyword_args) - - issues = response[:issues] - pagination = response[:pagination] - - validate_size(issues) - - handle_mapping_exceptions do - { - issues: map_to_errors(issues), - pagination: pagination - } - end - end - private - def validate_size(issues) - return if Gitlab::Utils::DeepSize.new(issues).valid? - - raise ResponseInvalidSizeError, "Sentry API response is too big. Limit is #{Gitlab::Utils::DeepSize.human_default_max_size}." + def api_urls + @api_urls ||= Sentry::ApiUrls.new(@url) end def handle_mapping_exceptions(&block) @@ -69,6 +35,7 @@ module Sentry def request_params { headers: { + 'Content-Type' => 'application/json', 'Authorization' => "Bearer #{@token}" }, follow_redirects: false @@ -76,43 +43,23 @@ module Sentry end def http_get(url, params = {}) - response = handle_request_exceptions do + http_request do Gitlab::HTTP.get(url, **request_params.merge(params)) end - handle_response(response) end - def get_issues(**keyword_args) - response = http_get( - issues_api_url, - query: list_issue_sentry_query(keyword_args) - ) - - { - issues: response[:body], - pagination: Sentry::PaginationParser.parse(response[:headers]) - } - end - - def list_issue_sentry_query(issue_status:, limit:, sort: nil, search_term: '', cursor: nil) - unless SENTRY_API_SORT_VALUE_MAP.key?(sort) - raise BadRequestError, 'Invalid value for sort param' + def http_put(url, params = {}) + http_request do + Gitlab::HTTP.put(url, **request_params.merge(body: params.to_json)) end - - { - query: "is:#{issue_status} #{search_term}".strip, - limit: limit, - sort: SENTRY_API_SORT_VALUE_MAP[sort], - cursor: cursor - }.compact end - def get_issue(issue_id:) - http_get(issue_api_url(issue_id))[:body] - end + def http_request + response = handle_request_exceptions do + yield + end - def get_issue_latest_event(issue_id:) - http_get(issue_latest_event_api_url(issue_id))[:body] + handle_response(response) end def handle_request_exceptions @@ -134,7 +81,7 @@ module Sentry end def handle_response(response) - unless response.code == 200 + unless response.code.between?(200, 204) raise_error "Sentry response status code: #{response.code}" end @@ -144,129 +91,5 @@ module Sentry def raise_error(message) raise Client::Error, message end - - def issue_api_url(issue_id) - issue_url = URI(@url) - issue_url.path = "/api/0/issues/#{issue_id}/" - - issue_url - end - - def issue_latest_event_api_url(issue_id) - latest_event_url = URI(@url) - latest_event_url.path = "/api/0/issues/#{issue_id}/events/latest/" - - latest_event_url - end - - def issues_api_url - issues_url = URI(@url + '/issues/') - issues_url.path.squeeze!('/') - - issues_url - end - - def map_to_errors(issues) - issues.map(&method(:map_to_error)) - end - - def issue_url(id) - issues_url = @url + "/issues/#{id}" - - parse_sentry_url(issues_url) - end - - def project_url - parse_sentry_url(@url) - end - - def parse_sentry_url(api_url) - url = ErrorTracking::ProjectErrorTrackingSetting.extract_sentry_external_url(api_url) - - uri = URI(url) - uri.path.squeeze!('/') - # Remove trailing slash - uri = uri.to_s.gsub(/\/\z/, '') - - uri - end - - def map_to_event(event) - stack_trace = parse_stack_trace(event) - - Gitlab::ErrorTracking::ErrorEvent.new( - issue_id: event.dig('groupID'), - date_received: event.dig('dateReceived'), - stack_trace_entries: stack_trace - ) - end - - def parse_stack_trace(event) - exception_entry = event.dig('entries')&.detect { |h| h['type'] == 'exception' } - return unless exception_entry - - exception_values = exception_entry.dig('data', 'values') - stack_trace_entry = exception_values&.detect { |h| h['stacktrace'].present? } - return unless stack_trace_entry - - stack_trace_entry.dig('stacktrace', 'frames') - end - - def parse_gitlab_issue(plugin_issues) - return unless plugin_issues - - gitlab_plugin = plugin_issues.detect { |item| item['id'] == 'gitlab' } - return unless gitlab_plugin - - gitlab_plugin.dig('issue', 'url') - end - - def map_to_detailed_error(issue) - Gitlab::ErrorTracking::DetailedError.new( - id: issue.fetch('id'), - first_seen: issue.fetch('firstSeen', nil), - last_seen: issue.fetch('lastSeen', nil), - title: issue.fetch('title', nil), - type: issue.fetch('type', nil), - user_count: issue.fetch('userCount', nil), - count: issue.fetch('count', nil), - message: issue.dig('metadata', 'value'), - culprit: issue.fetch('culprit', nil), - external_url: issue_url(issue.fetch('id')), - external_base_url: project_url, - short_id: issue.fetch('shortId', nil), - status: issue.fetch('status', nil), - frequency: issue.dig('stats', '24h'), - project_id: issue.dig('project', 'id'), - project_name: issue.dig('project', 'name'), - project_slug: issue.dig('project', 'slug'), - gitlab_issue: parse_gitlab_issue(issue.fetch('pluginIssues', nil)), - first_release_last_commit: issue.dig('firstRelease', 'lastCommit'), - last_release_last_commit: issue.dig('lastRelease', 'lastCommit'), - first_release_short_version: issue.dig('firstRelease', 'shortVersion'), - last_release_short_version: issue.dig('lastRelease', 'shortVersion') - ) - end - - def map_to_error(issue) - Gitlab::ErrorTracking::Error.new( - id: issue.fetch('id'), - first_seen: issue.fetch('firstSeen', nil), - last_seen: issue.fetch('lastSeen', nil), - title: issue.fetch('title', nil), - type: issue.fetch('type', nil), - user_count: issue.fetch('userCount', nil), - count: issue.fetch('count', nil), - message: issue.dig('metadata', 'value'), - culprit: issue.fetch('culprit', nil), - external_url: issue_url(issue.fetch('id')), - short_id: issue.fetch('shortId', nil), - status: issue.fetch('status', nil), - frequency: issue.dig('stats', '24h'), - project_id: issue.dig('project', 'id'), - project_name: issue.dig('project', 'name'), - project_slug: issue.dig('project', 'slug') - ) - end end end diff --git a/lib/sentry/client/event.rb b/lib/sentry/client/event.rb new file mode 100644 index 00000000000..01dfaa25969 --- /dev/null +++ b/lib/sentry/client/event.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Sentry + class Client + module Event + def issue_latest_event(issue_id:) + latest_event = http_get(api_urls.issue_latest_event_url(issue_id))[:body] + + map_to_event(latest_event) + end + + private + + def map_to_event(event) + stack_trace = parse_stack_trace(event) + + Gitlab::ErrorTracking::ErrorEvent.new( + issue_id: event.dig('groupID'), + date_received: event.dig('dateReceived'), + stack_trace_entries: stack_trace + ) + end + + def parse_stack_trace(event) + exception_entry = event.dig('entries')&.detect { |h| h['type'] == 'exception' } + return [] unless exception_entry + + exception_values = exception_entry.dig('data', 'values') + stack_trace_entry = exception_values&.detect { |h| h['stacktrace'].present? } + return [] unless stack_trace_entry + + stack_trace_entry.dig('stacktrace', 'frames') || [] + end + end + end +end diff --git a/lib/sentry/client/issue.rb b/lib/sentry/client/issue.rb new file mode 100644 index 00000000000..1c5d88e8862 --- /dev/null +++ b/lib/sentry/client/issue.rb @@ -0,0 +1,168 @@ +# frozen_string_literal: true + +module Sentry + class Client + module Issue + BadRequestError = Class.new(StandardError) + + SENTRY_API_SORT_VALUE_MAP = { + # <accepted_by_client> => <accepted_by_sentry_api> + 'frequency' => 'freq', + 'first_seen' => 'new', + 'last_seen' => nil + }.freeze + + def list_issues(**keyword_args) + response = get_issues(keyword_args) + + issues = response[:issues] + pagination = response[:pagination] + + validate_size(issues) + + handle_mapping_exceptions do + { + issues: map_to_errors(issues), + pagination: pagination + } + end + end + + def issue_details(issue_id:) + issue = get_issue(issue_id: issue_id) + + map_to_detailed_error(issue) + end + + def update_issue(issue_id:, params:) + http_put(api_urls.issue_url(issue_id), params)[:body] + end + + private + + def get_issues(**keyword_args) + response = http_get( + api_urls.issues_url, + query: list_issue_sentry_query(keyword_args) + ) + + { + issues: response[:body], + pagination: Sentry::PaginationParser.parse(response[:headers]) + } + end + + def list_issue_sentry_query(issue_status:, limit:, sort: nil, search_term: '', cursor: nil) + unless SENTRY_API_SORT_VALUE_MAP.key?(sort) + raise BadRequestError, 'Invalid value for sort param' + end + + { + query: "is:#{issue_status} #{search_term}".strip, + limit: limit, + sort: SENTRY_API_SORT_VALUE_MAP[sort], + cursor: cursor + }.compact + end + + def validate_size(issues) + return if Gitlab::Utils::DeepSize.new(issues).valid? + + raise ResponseInvalidSizeError, "Sentry API response is too big. Limit is #{Gitlab::Utils::DeepSize.human_default_max_size}." + end + + def get_issue(issue_id:) + http_get(api_urls.issue_url(issue_id))[:body] + end + + def parse_gitlab_issue(plugin_issues) + return unless plugin_issues + + gitlab_plugin = plugin_issues.detect { |item| item['id'] == 'gitlab' } + return unless gitlab_plugin + + gitlab_plugin.dig('issue', 'url') + end + + def issue_url(id) + parse_sentry_url("#{url}/issues/#{id}") + end + + def project_url + parse_sentry_url(url) + end + + def parse_sentry_url(api_url) + url = ErrorTracking::ProjectErrorTrackingSetting.extract_sentry_external_url(api_url) + + uri = URI(url) + uri.path.squeeze!('/') + # Remove trailing slash + uri = uri.to_s.gsub(/\/\z/, '') + + uri + end + + def map_to_errors(issues) + issues.map(&method(:map_to_error)) + end + + def map_to_error(issue) + Gitlab::ErrorTracking::Error.new( + id: issue.fetch('id'), + first_seen: issue.fetch('firstSeen', nil), + last_seen: issue.fetch('lastSeen', nil), + title: issue.fetch('title', nil), + type: issue.fetch('type', nil), + user_count: issue.fetch('userCount', nil), + count: issue.fetch('count', nil), + message: issue.dig('metadata', 'value'), + culprit: issue.fetch('culprit', nil), + external_url: issue_url(issue.fetch('id')), + short_id: issue.fetch('shortId', nil), + status: issue.fetch('status', nil), + frequency: issue.dig('stats', '24h'), + project_id: issue.dig('project', 'id'), + project_name: issue.dig('project', 'name'), + project_slug: issue.dig('project', 'slug') + ) + end + + def map_to_detailed_error(issue) + Gitlab::ErrorTracking::DetailedError.new({ + id: issue.fetch('id'), + first_seen: issue.fetch('firstSeen', nil), + last_seen: issue.fetch('lastSeen', nil), + tags: extract_tags(issue), + title: issue.fetch('title', nil), + type: issue.fetch('type', nil), + user_count: issue.fetch('userCount', nil), + count: issue.fetch('count', nil), + message: issue.dig('metadata', 'value'), + culprit: issue.fetch('culprit', nil), + external_url: issue_url(issue.fetch('id')), + external_base_url: project_url, + short_id: issue.fetch('shortId', nil), + status: issue.fetch('status', nil), + frequency: issue.dig('stats', '24h'), + gitlab_issue: parse_gitlab_issue(issue.fetch('pluginIssues', nil)), + project_id: issue.dig('project', 'id'), + project_name: issue.dig('project', 'name'), + project_slug: issue.dig('project', 'slug'), + first_release_last_commit: issue.dig('firstRelease', 'lastCommit'), + first_release_short_version: issue.dig('firstRelease', 'shortVersion'), + first_release_version: issue.dig('firstRelease', 'version'), + last_release_last_commit: issue.dig('lastRelease', 'lastCommit'), + last_release_short_version: issue.dig('lastRelease', 'shortVersion') + }) + end + + def extract_tags(issue) + { + level: issue.fetch('level', nil), + logger: issue.fetch('logger', nil) + } + end + end + end +end diff --git a/lib/sentry/client/issue_link.rb b/lib/sentry/client/issue_link.rb new file mode 100644 index 00000000000..200b1a6b435 --- /dev/null +++ b/lib/sentry/client/issue_link.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Sentry + class Client + module IssueLink + def create_issue_link(integration_id, sentry_issue_identifier, issue) + issue_link_url = issue_link_api_url(integration_id, sentry_issue_identifier) + + params = { + project: issue.project.id, + externalIssue: "#{issue.project.id}##{issue.iid}" + } + + http_put(issue_link_url, params) + end + + private + + def issue_link_api_url(integration_id, sentry_issue_identifier) + issue_link_url = URI(url) + issue_link_url.path = "/api/0/groups/#{sentry_issue_identifier}/integrations/#{integration_id}/" + + issue_link_url + end + end + end +end diff --git a/lib/sentry/client/projects.rb b/lib/sentry/client/projects.rb index 68f8fe0f9c9..e686d4ff715 100644 --- a/lib/sentry/client/projects.rb +++ b/lib/sentry/client/projects.rb @@ -14,14 +14,7 @@ module Sentry private def get_projects - http_get(projects_api_url)[:body] - end - - def projects_api_url - projects_url = URI(url) - projects_url.path = '/api/0/projects/' - - projects_url + http_get(api_urls.projects_url)[:body] end def map_to_projects(projects) diff --git a/lib/sentry/client/repo.rb b/lib/sentry/client/repo.rb new file mode 100644 index 00000000000..9a0ed3c7342 --- /dev/null +++ b/lib/sentry/client/repo.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Sentry + class Client + module Repo + def repos(organization_slug) + repos_url = repos_api_url(organization_slug) + + repos = http_get(repos_url)[:body] + + handle_mapping_exceptions do + map_to_repos(repos) + end + end + + private + + def repos_api_url(organization_slug) + repos_url = URI(url) + repos_url.path = "/api/0/organizations/#{organization_slug}/repos/" + + repos_url + end + + def map_to_repos(repos) + repos.map(&method(:map_to_repo)) + end + + def map_to_repo(repo) + Gitlab::ErrorTracking::Repo.new( + status: repo.fetch('status'), + integration_id: repo.fetch('integrationId'), + project_id: repo.fetch('externalSlug') + ) + end + end + end +end diff --git a/lib/tasks/plugins.rake b/lib/tasks/file_hooks.rake index e73dd7e68df..20a726de65b 100644 --- a/lib/tasks/plugins.rake +++ b/lib/tasks/file_hooks.rake @@ -1,10 +1,10 @@ -namespace :plugins do +namespace :file_hooks do desc 'Validate existing plugins' task validate: :environment do - puts 'Validating plugins from /plugins directory' + puts 'Validating file hooks from /plugins directory' - Gitlab::Plugin.files.each do |file| - success, message = Gitlab::Plugin.execute(file, Gitlab::DataBuilder::Push::SAMPLE_DATA) + Gitlab::FileHook.files.each do |file| + success, message = Gitlab::FileHook.execute(file, Gitlab::DataBuilder::Push::SAMPLE_DATA) if success puts "* #{file} succeed (zero exit code)." diff --git a/lib/tasks/gitlab/generate_sample_prometheus_data.rake b/lib/tasks/gitlab/generate_sample_prometheus_data.rake index a988494ca61..250eaaa5568 100644 --- a/lib/tasks/gitlab/generate_sample_prometheus_data.rake +++ b/lib/tasks/gitlab/generate_sample_prometheus_data.rake @@ -8,12 +8,17 @@ namespace :gitlab do sample_metrics_directory_name = Metrics::SampleMetricsService::DIRECTORY FileUtils.mkdir_p(sample_metrics_directory_name) + sample_metrics_intervals = [30.minutes, 180.minutes, 8.hours, 24.hours, 72.hours, 7.days] + metrics.each do |metric| query = metric.query % query_variables - result = environment.prometheus_adapter.prometheus_client.query_range(query, start: 7.days.ago) next unless metric.identifier + result = sample_metrics_intervals.each_with_object({}) do |interval, memo| + memo[interval.to_i / 60] = environment.prometheus_adapter.prometheus_client.query_range(query, start: interval.ago) + end + File.write("#{sample_metrics_directory_name}/#{metric.identifier}.yml", result.to_yaml) end end diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake index 4142903d9c3..6f11646c841 100644 --- a/lib/tasks/gitlab/lfs/migrate.rake +++ b/lib/tasks/gitlab/lfs/migrate.rake @@ -9,7 +9,6 @@ namespace :gitlab do LfsObject.with_files_stored_locally .find_each(batch_size: 10) do |lfs_object| - lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage") @@ -24,7 +23,6 @@ namespace :gitlab do LfsObject.with_files_stored_remotely .find_each(batch_size: 10) do |lfs_object| - lfs_object.file.migrate!(LfsObjectUploader::Store::LOCAL) logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to local storage") diff --git a/lib/tasks/pngquant.rake b/lib/tasks/pngquant.rake new file mode 100644 index 00000000000..56dfd5ed081 --- /dev/null +++ b/lib/tasks/pngquant.rake @@ -0,0 +1,97 @@ +return if Rails.env.production? + +require 'png_quantizator' +require 'parallel' + +# The amount of variance (in bytes) allowed in +# file size when testing for compression size +TOLERANCE = 10000 + +namespace :pngquant do + # Returns an array of all images eligible for compression + def doc_images + Dir.glob('doc/**/*.png', File::FNM_CASEFOLD) + end + + # Runs pngquant on an image and optionally + # writes the result to disk + def compress_image(file, overwrite_original) + compressed_file = "#{file}.compressed" + FileUtils.copy(file, compressed_file) + + pngquant_file = PngQuantizator::Image.new(compressed_file) + + # Run the image repeatedly through pngquant until + # the change in file size is within TOLERANCE + loop do + before = File.size(compressed_file) + pngquant_file.quantize! + after = File.size(compressed_file) + break if before - after <= TOLERANCE + end + + savings = File.size(file) - File.size(compressed_file) + is_uncompressed = savings > TOLERANCE + + if is_uncompressed && overwrite_original + FileUtils.copy(compressed_file, file) + end + + FileUtils.remove(compressed_file) + + [is_uncompressed, savings] + end + + # Ensures pngquant is available and prints an error if not + def check_executable + unless system('pngquant --version', out: File::NULL) + warn( + 'Error: pngquant executable was not detected in the system.'.color(:red), + 'Download pngquant at https://pngquant.org/ and place the executable in /usr/local/bin'.color(:green) + ) + abort + end + end + + desc 'GitLab | pngquant | Compress all documentation PNG images using pngquant' + task :compress do + check_executable + + files = doc_images + puts "Compressing #{files.size} PNG files in doc/**" + + Parallel.each(files) do |file| + was_uncompressed, savings = compress_image(file, true) + + if was_uncompressed + puts "#{file} was reduced by #{savings} bytes" + end + end + end + + desc 'GitLab | pngquant | Checks that all documentation PNG images have been compressed with pngquant' + task :lint do + check_executable + + files = doc_images + puts "Checking #{files.size} PNG files in doc/**" + + uncompressed_files = Parallel.map(files) do |file| + is_uncompressed, _ = compress_image(file, false) + if is_uncompressed + puts "Uncompressed file detected: ".color(:red) + file + file + end + end.compact + + if uncompressed_files.empty? + puts "All documentation images are optimally compressed!".color(:green) + else + warn( + "The #{uncompressed_files.size} image(s) above have not been optimally compressed using pngquant.".color(:red), + 'Please run "bin/rake pngquant:compress" and commit the result.' + ) + abort + end + end +end diff --git a/lib/tasks/sidekiq.rake b/lib/tasks/sidekiq.rake index dd9ce86f7ca..cb9f4c751ed 100644 --- a/lib/tasks/sidekiq.rake +++ b/lib/tasks/sidekiq.rake @@ -1,21 +1,38 @@ namespace :sidekiq do - desc "GitLab | Stop sidekiq" + def deprecation_warning! + warn <<~WARNING + This task is deprecated and will be removed in 13.0 as it is thought to be unused. + + If you are using this task, please comment on the below issue: + https://gitlab.com/gitlab-org/gitlab/issues/196731 + WARNING + end + + desc "[DEPRECATED] GitLab | Stop sidekiq" task :stop do + deprecation_warning! + system(*%w(bin/background_jobs stop)) end - desc "GitLab | Start sidekiq" + desc "[DEPRECATED] GitLab | Start sidekiq" task :start do + deprecation_warning! + system(*%w(bin/background_jobs start)) end - desc 'GitLab | Restart sidekiq' + desc '[DEPRECATED] GitLab | Restart sidekiq' task :restart do + deprecation_warning! + system(*%w(bin/background_jobs restart)) end - desc "GitLab | Start sidekiq with launchd on Mac OS X" + desc "[DEPRECATED] GitLab | Start sidekiq with launchd on Mac OS X" task :launchd do + deprecation_warning! + system(*%w(bin/background_jobs start_no_deamonize)) end end |