diff options
Diffstat (limited to 'lib')
405 files changed, 7145 insertions, 2867 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 7016a66593d..219ed45eff6 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -10,14 +10,15 @@ module API NAMESPACE_OR_PROJECT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze COMMIT_ENDPOINT_REQUIREMENTS = NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze USER_REQUIREMENTS = { user_id: NO_SLASH_URL_PART_REGEX }.freeze + LOG_FILTERS = ::Rails.application.config.filter_parameters + [/^output$/] insert_before Grape::Middleware::Error, GrapeLogging::Middleware::RequestLogger, logger: Logger.new(LOG_FILENAME), formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, include: [ - GrapeLogging::Loggers::FilterParameters.new, - GrapeLogging::Loggers::ClientEnv.new, + GrapeLogging::Loggers::FilterParameters.new(LOG_FILTERS), + Gitlab::GrapeLogging::Loggers::ClientEnvLogger.new, Gitlab::GrapeLogging::Loggers::RouteLogger.new, Gitlab::GrapeLogging::Loggers::UserLogger.new, Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new, @@ -103,7 +104,6 @@ module API mount ::API::BroadcastMessages mount ::API::Commits mount ::API::CommitStatuses - mount ::API::ContainerRegistry mount ::API::DeployKeys mount ::API::Deployments mount ::API::Environments @@ -111,9 +111,11 @@ module API mount ::API::Features mount ::API::Files mount ::API::GroupBoards + mount ::API::GroupClusters mount ::API::GroupLabels mount ::API::GroupMilestones mount ::API::Groups + mount ::API::GroupContainerRepositories mount ::API::GroupVariables mount ::API::ImportGithub mount ::API::Internal @@ -136,6 +138,7 @@ module API mount ::API::Pipelines mount ::API::PipelineSchedules mount ::API::ProjectClusters + mount ::API::ProjectContainerRepositories mount ::API::ProjectEvents mount ::API::ProjectExport mount ::API::ProjectImport diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index a1851ba3627..89b7e5c5e4b 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -69,12 +69,12 @@ module API post endpoint do not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? - award = awardable.create_award_emoji(params[:name], current_user) + service = AwardEmojis::AddService.new(awardable, params[:name], current_user).execute - if award.persisted? - present award, with: Entities::AwardEmoji + if service[:status] == :success + present service[:award], with: Entities::AwardEmoji else - not_found!("Award Emoji #{award.errors.messages}") + not_found!("Award Emoji #{service[:message]}") end end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 08b4f8db8b0..d58a5e214ed 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -52,6 +52,7 @@ module API optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' optional :coverage, type: Float, desc: 'The total code coverage' + optional :pipeline_id, type: Integer, desc: 'An existing pipeline ID, when multiple pipelines on the same commit SHA have been triggered' end # rubocop: disable CodeReuse/ActiveRecord post ':id/statuses/:sha' do @@ -73,7 +74,8 @@ module API name = params[:name] || params[:context] || 'default' - pipeline = @project.pipeline_for(ref, commit.sha) + pipeline = @project.pipeline_for(ref, commit.sha, params[:pipeline_id]) + unless pipeline pipeline = @project.ci_pipelines.create!( source: :external, diff --git a/lib/api/commits.rb b/lib/api/commits.rb index eebded87ebc..a2f3e87ebd2 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -43,7 +43,7 @@ module API path = params[:path] before = params[:until] after = params[:since] - ref = params[:ref_name] || user_project.try(:default_branch) || 'master' unless params[:all] + ref = params[:ref_name].presence || user_project.try(:default_branch) || 'master' unless params[:all] offset = (params[:page] - 1) * params[:per_page] all = params[:all] with_stats = params[:with_stats] @@ -76,7 +76,7 @@ module API detail 'This feature was introduced in GitLab 8.13' end params do - requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false + requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide either `start_branch` or `start_sha`, and optionally `start_project`.', allow_blank: false requires :commit_message, type: String, desc: 'Commit message' requires :actions, type: Array, desc: 'Actions to perform in commit' do requires :action, type: String, desc: 'The action to perform, `create`, `delete`, `move`, `update`, `chmod`', values: %w[create update move delete chmod].freeze @@ -98,12 +98,16 @@ module API requires :execute_filemode, type: Boolean, desc: 'When `true/false` enables/disables the execute flag on the file.' end end - optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' - optional :start_project, types: [Integer, String], desc: 'The ID or path of the project to start the commit from' + + optional :start_branch, type: String, desc: 'Name of the branch to start the new branch from' + optional :start_sha, type: String, desc: 'SHA of the commit to start the new branch from' + mutually_exclusive :start_branch, :start_sha + + optional :start_project, types: [Integer, String], desc: 'The ID or path of the project to start the new branch from' optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' optional :stats, type: Boolean, default: true, desc: 'Include commit stats' - optional :force, type: Boolean, default: false, desc: 'When `true` overwrites the target branch with a new commit based on the `start_branch`' + optional :force, type: Boolean, default: false, desc: 'When `true` overwrites the target branch with a new commit based on the `start_branch` or `start_sha`' end post ':id/repository/commits' do if params[:start_project] @@ -118,7 +122,7 @@ module API attrs = declared_params attrs[:branch_name] = attrs.delete(:branch) - attrs[:start_branch] ||= attrs[:branch_name] + attrs[:start_branch] ||= attrs[:branch_name] unless attrs[:start_sha] attrs[:start_project] = start_project if start_project result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -126,7 +130,7 @@ module API if result[:status] == :success commit_detail = user_project.repository.commit(result[:result]) - Gitlab::WebIdeCommitsCounter.increment if find_user_from_warden + Gitlab::UsageDataCounters::WebIdeCounter.increment_commits_count if find_user_from_warden present commit_detail, with: Entities::CommitDetail, stats: params[:stats] else diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index cc62ce22a1b..6c1acc3963f 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -4,6 +4,7 @@ module API class Discussions < Grape::API include PaginationParams helpers ::API::Helpers::NotesHelpers + helpers ::RendersNotes before { authenticate! } @@ -23,21 +24,15 @@ module API requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' use :pagination end - # rubocop: disable CodeReuse/ActiveRecord + get ":id/#{noteables_path}/:noteable_id/discussions" do noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) - notes = noteable.notes - .inc_relations_for_view - .includes(:noteable) - .fresh - - notes = notes.reject { |n| n.cross_reference_not_visible_for?(current_user) } + notes = readable_discussion_notes(noteable) discussions = Kaminari.paginate_array(Discussion.build_collection(notes, noteable)) present paginate(discussions), with: Entities::Discussion end - # rubocop: enable CodeReuse/ActiveRecord desc "Get a single #{noteable_type.to_s.downcase} discussion" do success Entities::Discussion @@ -226,13 +221,24 @@ module API helpers do # rubocop: disable CodeReuse/ActiveRecord - def readable_discussion_notes(noteable, discussion_id) + def readable_discussion_notes(noteable, discussion_id = nil) notes = noteable.notes - .where(discussion_id: discussion_id) + notes = notes.where(discussion_id: discussion_id) if discussion_id + notes = notes .inc_relations_for_view .includes(:noteable) .fresh + # Without RendersActions#prepare_notes_for_rendering, + # Note#cross_reference_not_visible_for? will attempt to render + # Markdown references mentioned in the note to see whether they + # should be redacted. For notes that reference a commit, this + # would also incur a Gitaly call to verify the commit exists. + # + # With prepare_notes_for_rendering, we can avoid Gitaly calls + # because notes are redacted if they point to projects that + # cannot be accessed by the user. + notes = prepare_notes_for_rendering(notes) notes.reject { |n| n.cross_reference_not_visible_for?(current_user) } end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 765819e6bf1..6f86adf6a5c 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -2,6 +2,19 @@ module API module Entities + class BlameRangeCommit < Grape::Entity + expose :id + expose :parent_ids + expose :message + expose :authored_date, :author_name, :author_email + expose :committed_date, :committer_name, :committer_email + end + + class BlameRange < Grape::Entity + expose :commit, using: BlameRangeCommit + expose :lines + end + class WikiPageBasic < Grape::Entity expose :format expose :slug @@ -64,6 +77,11 @@ module API expose :last_activity_on, as: :last_activity_at # Back-compat end + class UserStarsProject < Grape::Entity + expose :starred_since + expose :user, using: Entities::UserBasic + end + class Identity < Grape::Entity expose :provider, :extern_uid end @@ -294,7 +312,6 @@ module API expose :statistics, using: 'API::Entities::ProjectStatistics', if: -> (project, options) { options[:statistics] && Ability.allowed?(options[:current_user], :read_statistics, project) } - expose :external_authorization_classification_label expose :auto_devops_enabled?, as: :auto_devops_enabled expose :auto_devops_deploy_strategy do |project, options| project.auto_devops.nil? ? 'continuous' : project.auto_devops.deploy_strategy @@ -367,10 +384,7 @@ module API end expose :request_access_enabled expose :full_name, :full_path - - if ::Group.supports_nested_objects? - expose :parent_id - end + expose :parent_id expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes @@ -631,7 +645,10 @@ module API end end - expose :subscribed do |issue, options| + # Calculating the value of subscribed field triggers Markdown + # processing. We can't do that for multiple issues / merge + # requests in a single API request. + expose :subscribed, if: -> (_, options) { options.fetch(:include_subscribed, true) } do |issue, options| issue.subscribed?(options[:current_user], options[:project] || issue.project) end end @@ -1053,15 +1070,8 @@ module API # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) relation = super(projects_relation, options) - - # MySQL doesn't support LIMIT inside an IN subquery - if Gitlab::Database.mysql? - project_ids = relation.pluck('projects.id') - namespace_ids = relation.pluck(:namespace_id) - else - project_ids = relation.select('projects.id') - namespace_ids = relation.select(:namespace_id) - end + project_ids = relation.select('projects.id') + namespace_ids = relation.select(:namespace_id) options[:project_members] = options[:current_user] .project_members @@ -1083,16 +1093,18 @@ module API end class Label < LabelBasic - expose :open_issues_count do |label, options| - label.open_issues_count(options[:current_user]) - end + with_options if: lambda { |_, options| options[:with_counts] } do + expose :open_issues_count do |label, options| + label.open_issues_count(options[:current_user]) + end - expose :closed_issues_count do |label, options| - label.closed_issues_count(options[:current_user]) - end + expose :closed_issues_count do |label, options| + label.closed_issues_count(options[:current_user]) + end - expose :open_merge_requests_count do |label, options| - label.open_merge_requests_count(options[:current_user]) + expose :open_merge_requests_count do |label, options| + label.open_merge_requests_count(options[:current_user]) + end end expose :subscribed do |label, options| @@ -1160,6 +1172,7 @@ module API attributes = ::ApplicationSettingsHelper.visible_attributes attributes.delete(:performance_bar_allowed_group_path) attributes.delete(:performance_bar_enabled) + attributes.delete(:allow_local_requests_from_hooks_and_services) attributes end @@ -1178,6 +1191,7 @@ module API # support legacy names, can be removed in v5 expose :password_authentication_enabled_for_web, as: :password_authentication_enabled expose :password_authentication_enabled_for_web, as: :signin_enabled + expose :allow_local_requests_from_web_hooks_and_services, as: :allow_local_requests_from_hooks_and_services end # deprecated old Release representation @@ -1337,6 +1351,7 @@ module API expose :variable_type, :key, :value expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) } expose :masked?, as: :masked, if: -> (entity, _) { entity.respond_to?(:masked?) } + expose :environment_scope, if: -> (entity, _) { entity.respond_to?(:environment_scope) } end class Pipeline < PipelineBasic @@ -1686,5 +1701,27 @@ module API class ClusterProject < Cluster expose :project, using: Entities::BasicProjectDetails end + + class ClusterGroup < Cluster + expose :group, using: Entities::BasicGroupDetails + end end end + +# rubocop: disable Cop/InjectEnterpriseEditionModule +API::Entities.prepend_if_ee('EE::API::Entities::Entities') +::API::Entities::ApplicationSetting.prepend_if_ee('EE::API::Entities::ApplicationSetting') +::API::Entities::Board.prepend_if_ee('EE::API::Entities::Board') +::API::Entities::Group.prepend_if_ee('EE::API::Entities::Group', with_descendants: true) +::API::Entities::GroupDetail.prepend_if_ee('EE::API::Entities::GroupDetail') +::API::Entities::IssueBasic.prepend_if_ee('EE::API::Entities::IssueBasic', with_descendants: true) +::API::Entities::List.prepend_if_ee('EE::API::Entities::List') +::API::Entities::MergeRequestBasic.prepend_if_ee('EE::API::Entities::MergeRequestBasic', with_descendants: true) +::API::Entities::Namespace.prepend_if_ee('EE::API::Entities::Namespace') +::API::Entities::Project.prepend_if_ee('EE::API::Entities::Project', with_descendants: true) +::API::Entities::ProtectedRefAccess.prepend_if_ee('EE::API::Entities::ProtectedRefAccess') +::API::Entities::UserPublic.prepend_if_ee('EE::API::Entities::UserPublic', with_descendants: true) +::API::Entities::Todo.prepend_if_ee('EE::API::Entities::Todo') +::API::Entities::ProtectedBranch.prepend_if_ee('EE::API::Entities::ProtectedBranch') +::API::Entities::Identity.prepend_if_ee('EE::API::Entities::Identity') +::API::Entities::UserWithAdmin.prepend_if_ee('EE::API::Entities::UserWithAdmin', with_descendants: true) diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb index 00833ca7480..6250f35c7cb 100644 --- a/lib/api/entities/container_registry.rb +++ b/lib/api/entities/container_registry.rb @@ -3,18 +3,20 @@ module API module Entities module ContainerRegistry - class Repository < Grape::Entity - expose :id + class Tag < Grape::Entity expose :name expose :path expose :location - expose :created_at end - class Tag < Grape::Entity + class Repository < Grape::Entity + expose :id expose :name expose :path + expose :project_id expose :location + expose :created_at + expose :tags, using: Tag, if: -> (_, options) { options[:tags] } end class TagDetails < Tag diff --git a/lib/api/files.rb b/lib/api/files.rb index ca59d330e1c..0b438fb5bbc 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -83,6 +83,31 @@ module API resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do allow_access_with_scope :read_repository, if: -> (request) { request.get? || request.head? } + desc 'Get blame file metadata from repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + end + head ":id/repository/files/:file_path/blame", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + end + + desc 'Get blame file from the repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + end + get ":id/repository/files/:file_path/blame", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + + blame_ranges = Gitlab::Blame.new(@blob, @commit).groups(highlight: false) + present blame_ranges, with: Entities::BlameRange + end + desc 'Get raw file metadata from repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb new file mode 100644 index 00000000000..db0f8081140 --- /dev/null +++ b/lib/api/group_clusters.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module API + class GroupClusters < Grape::API + include PaginationParams + + before { authenticate! } + + # EE::API::GroupClusters will + # override these methods + helpers do + params :create_params_ee do + end + + params :update_params_ee do + end + end + + params do + requires :id, type: String, desc: 'The ID of the group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get all clusters from the group' do + success Entities::Cluster + end + params do + use :pagination + end + get ':id/clusters' do + authorize! :read_cluster, user_group + + present paginate(clusters_for_current_user), with: Entities::Cluster + end + + desc 'Get specific cluster for the group' do + success Entities::ClusterGroup + end + params do + requires :cluster_id, type: Integer, desc: 'The cluster ID' + end + get ':id/clusters/:cluster_id' do + authorize! :read_cluster, cluster + + present cluster, with: Entities::ClusterGroup + end + + desc 'Adds an existing cluster' do + success Entities::ClusterGroup + end + params do + requires :name, type: String, desc: 'Cluster name' + optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true' + optional :domain, type: String, desc: 'Cluster base domain' + optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true' + requires :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do + requires :api_url, type: String, allow_blank: false, desc: 'URL to access the Kubernetes API' + requires :token, type: String, desc: 'Token to authenticate against Kubernetes' + optional :ca_cert, type: String, desc: 'TLS certificate (needed if API is using a self-signed TLS certificate)' + optional :namespace, type: String, desc: 'Unique namespace related to Group' + optional :authorization_type, type: String, values: Clusters::Platforms::Kubernetes.authorization_types.keys, default: 'rbac', desc: 'Cluster authorization type, defaults to RBAC' + end + use :create_params_ee + end + post ':id/clusters/user' do + authorize! :add_cluster, user_group + + user_cluster = ::Clusters::CreateService + .new(current_user, create_cluster_user_params) + .execute + + if user_cluster.persisted? + present user_cluster, with: Entities::ClusterGroup + else + render_validation_error!(user_cluster) + end + end + + desc 'Update an existing cluster' do + success Entities::ClusterGroup + end + params do + requires :cluster_id, type: Integer, desc: 'The cluster ID' + optional :name, type: String, desc: 'Cluster name' + optional :domain, type: String, desc: 'Cluster base domain' + optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do + optional :api_url, type: String, desc: 'URL to access the Kubernetes API' + optional :token, type: String, desc: 'Token to authenticate against Kubernetes' + optional :ca_cert, type: String, desc: 'TLS certificate (needed if API is using a self-signed TLS certificate)' + optional :namespace, type: String, desc: 'Unique namespace related to Group' + end + use :update_params_ee + end + put ':id/clusters/:cluster_id' do + authorize! :update_cluster, cluster + + update_service = Clusters::UpdateService.new(current_user, update_cluster_params) + + if update_service.execute(cluster) + present cluster, with: Entities::ClusterGroup + else + render_validation_error!(cluster) + end + end + + desc 'Remove a cluster' do + success Entities::ClusterGroup + end + params do + requires :cluster_id, type: Integer, desc: 'The Cluster ID' + end + delete ':id/clusters/:cluster_id' do + authorize! :admin_cluster, cluster + + destroy_conditionally!(cluster) + end + end + + helpers do + def clusters_for_current_user + @clusters_for_current_user ||= ClustersFinder.new(user_group, current_user, :all).execute + end + + def cluster + @cluster ||= clusters_for_current_user.find(params[:cluster_id]) + end + + def create_cluster_user_params + declared_params.merge({ + provider_type: :user, + platform_type: :kubernetes, + clusterable: user_group + }) + end + + def update_cluster_params + declared_params(include_missing: false).without(:cluster_id) + end + end + end +end diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb new file mode 100644 index 00000000000..fd24662cc9a --- /dev/null +++ b/lib/api/group_container_repositories.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module API + class GroupContainerRepositories < Grape::API + include PaginationParams + + before { authorize_read_group_container_images! } + + REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( + tag_name: API::NO_SLASH_URL_PART_REGEX) + + params do + requires :id, type: String, desc: "Group's ID or path" + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get a list of all repositories within a group' do + detail 'This feature was introduced in GitLab 12.2.' + success Entities::ContainerRegistry::Repository + end + params do + use :pagination + optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included' + end + get ':id/registry/repositories' do + repositories = ContainerRepositoriesFinder.new( + id: user_group.id, container_type: :group + ).execute + + present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags] + end + end + + helpers do + def authorize_read_group_container_images! + authorize! :read_container_image, user_group + end + end + end +end diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb index 0dbc5f45a68..79a44941c81 100644 --- a/lib/api/group_labels.rb +++ b/lib/api/group_labels.rb @@ -16,6 +16,8 @@ module API success Entities::GroupLabel end params do + optional :with_counts, type: Boolean, default: false, + desc: 'Include issue and merge request counts' use :pagination end get ':id/labels' do diff --git a/lib/api/groups.rb b/lib/api/groups.rb index ec1020c7c78..f545f33c06b 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -114,10 +114,7 @@ module API params do requires :name, type: String, desc: 'The name of the group' requires :path, type: String, desc: 'The path of the group' - - if ::Group.supports_nested_objects? - optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' - end + optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' use :optional_params end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 8ae42c6dadd..1aa6dc44bf7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -544,5 +544,9 @@ module API params[:archived] end + + def ip_address + env["action_dispatch.remote_ip"].to_s || request.ip + end end end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index c318f5b9127..9afe6c5b027 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -72,7 +72,7 @@ module API result == 'PONG' rescue => e - Rails.logger.warn("GitLab: An unexpected error occurred in pinging to Redis: #{e}") + Rails.logger.warn("GitLab: An unexpected error occurred in pinging to Redis: #{e}") # rubocop:disable Gitlab/RailsLogger false end diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb index 5b7199fddb0..a8480bb9339 100644 --- a/lib/api/helpers/issues_helpers.rb +++ b/lib/api/helpers/issues_helpers.rb @@ -27,6 +27,10 @@ module API ] end + def self.sort_options + %w[created_at updated_at priority due_date relative_position label_priority milestone_due popularity] + end + def issue_finder(args = {}) args = declared_params.merge(args) @@ -34,15 +38,14 @@ module API args[:milestone_title] ||= args.delete(:milestone) args[:label_name] ||= args.delete(:labels) args[:scope] = args[:scope].underscore if args[:scope] + args[:sort] = "#{args[:order_by]}_#{args[:sort]}" IssuesFinder.new(current_user, args) end def find_issues(args = {}) finder = issue_finder(args) - issues = finder.execute.with_api_entity_associations - - issues.reorder(order_options_with_tie_breaker) # rubocop: disable CodeReuse/ActiveRecord + finder.execute.with_api_entity_associations end def issues_statistics(args = {}) diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb index c11e7d614ab..ec5b688dd1c 100644 --- a/lib/api/helpers/label_helpers.rb +++ b/lib/api/helpers/label_helpers.rb @@ -11,15 +11,19 @@ module API optional :description, type: String, desc: 'The description of label to be created' end - def find_label(parent, id, include_ancestor_groups: true) + def find_label(parent, id_or_title, include_ancestor_groups: true) labels = available_labels_for(parent, include_ancestor_groups: include_ancestor_groups) - label = labels.find_by_id(id) || labels.find_by_title(id) + label = labels.find_by_id(id_or_title) || labels.find_by_title(id_or_title) label || not_found!('Label') end def get_labels(parent, entity) - present paginate(available_labels_for(parent)), with: entity, current_user: current_user, parent: parent + present paginate(available_labels_for(parent)), + with: entity, + current_user: current_user, + parent: parent, + with_counts: params[:with_counts] end def create_label(parent, entity) @@ -31,12 +35,7 @@ module API priority = params.delete(:priority) label_params = declared_params(include_missing: false) - label = - if parent.is_a?(Project) - ::Labels::CreateService.new(label_params).execute(project: parent) - else - ::Labels::CreateService.new(label_params).execute(group: parent) - end + label = ::Labels::CreateService.new(label_params).execute(create_service_params(parent)) if label.persisted? if parent.is_a?(Project) @@ -52,10 +51,13 @@ module API def update_label(parent, entity) authorize! :admin_label, parent - label = find_label(parent, params[:name], include_ancestor_groups: false) + label = find_label(parent, params_id_or_title, include_ancestor_groups: false) update_priority = params.key?(:priority) priority = params.delete(:priority) + # params is used to update the label so we need to remove this field here + params.delete(:label_id) + label = ::Labels::UpdateService.new(declared_params(include_missing: false)).execute(label) render_validation_error!(label) unless label.valid? @@ -73,10 +75,24 @@ module API def delete_label(parent) authorize! :admin_label, parent - label = find_label(parent, params[:name], include_ancestor_groups: false) + label = find_label(parent, params_id_or_title, include_ancestor_groups: false) destroy_conditionally!(label) end + + def params_id_or_title + @params_id_or_title ||= params[:label_id] || params[:name] + end + + def create_service_params(parent) + if parent.is_a?(Project) + { project: parent } + elsif parent.is_a?(Group) + { group: parent } + else + raise TypeError, 'Parent type is not supported' + end + end end end end diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index b03ac7deb71..b2bf6bf7417 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -3,6 +3,8 @@ module API module Helpers module NotesHelpers + include ::RendersNotes + def self.noteable_types # This is a method instead of a constant, allowing EE to more easily # extend it. @@ -74,14 +76,14 @@ module API end def find_noteable(parent_type, parent_id, noteable_type, noteable_id) - params = params_by_noteable_type_and_id(noteable_type, noteable_id) + params = finder_params_by_noteable_type_and_id(noteable_type, noteable_id, parent_id) - noteable = NotesFinder.new(user_project, current_user, params).target + noteable = NotesFinder.new(current_user, params).target noteable = nil unless can?(current_user, noteable_read_ability_name(noteable), noteable) noteable || not_found!(noteable_type) end - def params_by_noteable_type_and_id(type, id) + def finder_params_by_noteable_type_and_id(type, id, parent_id) target_type = type.name.underscore { target_type: target_type }.tap do |h| if %w(issue merge_request).include?(target_type) @@ -89,9 +91,15 @@ module API else h[:target_id] = id end + + add_parent_to_finder_params(h, type, parent_id) end end + def add_parent_to_finder_params(finder_params, noteable_type, parent_id) + finder_params[:project] = user_project + end + def noteable_parent(noteable) public_send("user_#{noteable.class.parent_class.to_s.underscore}") # rubocop:disable GitlabSecurity/PublicSend end diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 2a9b17ad22a..71bbc218f94 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -205,7 +205,9 @@ module API limited_total_count = pagination_data.total_count_with_limit if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT - pagination_data.without_count + # The call to `total_count_with_limit` memoizes `@arel` because of a call to `references_eager_loaded_tables?` + # We need to call `reset` because `without_count` relies on `@arel` being unmemoized + pagination_data.reset.without_count else pagination_data end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 0e21a7a66fd..51b7cf05c8f 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -42,7 +42,6 @@ module API optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" - optional :external_authorization_classification_label, type: String, desc: 'The classification label for the project' optional :ci_default_git_depth, type: Integer, desc: 'Default number of revisions for shallow cloning' optional :auto_devops_enabled, type: Boolean, desc: 'Flag indication if Auto DevOps is enabled' optional :auto_devops_deploy_strategy, type: String, values: %w(continuous manual timed_incremental), desc: 'Auto Deploy strategy' @@ -72,6 +71,7 @@ module API :build_timeout, :builds_access_level, :ci_config_path, + :ci_default_git_depth, :container_registry_enabled, :default_branch, :description, @@ -94,7 +94,6 @@ module API :visibility, :wiki_access_level, :avatar, - :external_authorization_classification_label, # TODO: remove in API v5, replaced by *_access_level :issues_enabled, @@ -105,6 +104,9 @@ module API :snippets_enabled ] end + + def filter_attributes_using_license!(attrs) + end end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index ff73a49d5e8..5b87eccf860 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -7,8 +7,7 @@ module API JOB_TOKEN_PARAM = :token def runner_registration_token_valid? - ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], - Gitlab::CurrentSettings.runners_registration_token) + ActiveSupport::SecurityUtils.secure_compare(params[:token], Gitlab::CurrentSettings.runners_registration_token) end def authenticate_runner! @@ -26,7 +25,7 @@ module API end def get_runner_ip - { ip_address: env["action_dispatch.remote_ip"].to_s || request.ip } + { ip_address: ip_address } end def current_runner diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb index c4ecf55969c..422db5c7a50 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/services_helpers.rb @@ -489,32 +489,6 @@ module API desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the Jira workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`' } ], - 'kubernetes' => [ - { - required: true, - name: :namespace, - type: String, - desc: 'The Kubernetes namespace to use' - }, - { - required: true, - name: :api_url, - type: String, - desc: 'The URL to the Kubernetes cluster API, e.g., https://kubernetes.example.com' - }, - { - required: true, - name: :token, - type: String, - desc: 'The service token to authenticate against the Kubernetes cluster with' - }, - { - required: false, - name: :ca_pem, - type: String, - desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' - } - ], 'mattermost-slash-commands' => [ { required: true, @@ -739,7 +713,6 @@ module API ::HipchatService, ::IrkerService, ::JiraService, - ::KubernetesService, ::MattermostSlashCommandsService, ::SlackSlashCommandsService, ::PackagistService, diff --git a/lib/api/helpers/variables_helpers.rb b/lib/api/helpers/variables_helpers.rb deleted file mode 100644 index 78a92d0f5a6..00000000000 --- a/lib/api/helpers/variables_helpers.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module API - module Helpers - module VariablesHelpers - extend ActiveSupport::Concern - extend Grape::API::Helpers - - params :optional_params_ee do - end - end - end -end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index d687acf3423..215178478d0 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -44,7 +44,7 @@ module API optional :with_labels_details, type: Boolean, desc: 'Return more label data than just lable title', default: false optional :state, type: String, values: %w[opened closed all], default: 'all', desc: 'Return opened, closed, or all issues' - optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + optional :order_by, type: String, values: Helpers::IssuesHelpers.sort_options, default: 'created_at', desc: 'Return issues ordered by `created_at` or `updated_at` fields.' optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return issues sorted in `asc` or `desc` order.' @@ -96,7 +96,8 @@ module API with: Entities::Issue, with_labels_details: declared_params[:with_labels_details], current_user: current_user, - issuable_metadata: issuable_meta_data(issues, 'Issue', current_user) + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user), + include_subscribed: false } present issues, options @@ -122,7 +123,8 @@ module API with: Entities::Issue, with_labels_details: declared_params[:with_labels_details], current_user: current_user, - issuable_metadata: issuable_meta_data(issues, 'Issue', current_user) + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user), + include_subscribed: false } present issues, options @@ -161,7 +163,8 @@ module API with_labels_details: declared_params[:with_labels_details], current_user: current_user, project: user_project, - issuable_metadata: issuable_meta_data(issues, 'Issue', current_user) + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user), + include_subscribed: false } present issues, options diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 456d595e637..63a5bd40478 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -27,7 +27,7 @@ module API requirements: { ref_name: /.+/ } do authorize_download_artifacts! - latest_build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) + latest_build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) present_carrierwave_file!(latest_build.artifacts_file) end @@ -45,7 +45,7 @@ module API requirements: { ref_name: /.+/ } do authorize_download_artifacts! - build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) + build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) path = Gitlab::Ci::Build::Artifacts::Path .new(params[:artifact_path]) diff --git a/lib/api/labels.rb b/lib/api/labels.rb index d729d3ee625..de89e94b0c0 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -15,6 +15,8 @@ module API success Entities::ProjectLabel end params do + optional :with_counts, type: Boolean, default: false, + desc: 'Include issue and merge request counts' use :pagination end get ':id/labels' do @@ -36,11 +38,13 @@ module API success Entities::ProjectLabel end params do - requires :name, type: String, desc: 'The name of the label to be updated' + optional :label_id, type: Integer, desc: 'The id of the label to be updated' + optional :name, type: String, desc: 'The name of the label to be updated' optional :new_name, type: String, desc: 'The new name of the label' optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" optional :description, type: String, desc: 'The new description of label' optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true + exactly_one_of :label_id, :name at_least_one_of :new_name, :color, :description, :priority end put ':id/labels' do @@ -51,11 +55,38 @@ module API success Entities::ProjectLabel end params do - requires :name, type: String, desc: 'The name of the label to be deleted' + optional :label_id, type: Integer, desc: 'The id of the label to be deleted' + optional :name, type: String, desc: 'The name of the label to be deleted' + exactly_one_of :label_id, :name end delete ':id/labels' do delete_label(user_project) end + + desc 'Promote a label to a group label' do + detail 'This feature was added in GitLab 12.3' + success Entities::GroupLabel + end + params do + requires :name, type: String, desc: 'The name of the label to be promoted' + end + put ':id/labels/promote' do + authorize! :admin_label, user_project + + label = find_label(user_project, params[:name], include_ancestor_groups: false) + + begin + group_label = ::Labels::PromoteService.new(user_project, current_user).execute(label) + + if group_label + present group_label, with: Entities::GroupLabel, current_user: current_user, parent: user_project.group + else + render_api_error!('Failed to promote project label to group label', 400) + end + rescue => error + render_api_error!(error.to_s, 400) + end + end end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 9381f045144..84563d66ee8 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -36,12 +36,13 @@ module API # page can have less elements than :per_page even if # there's more than one page. raw_notes = noteable.notes.with_metadata.reorder(order_options_with_tie_breaker) - notes = - # paginate() only works with a relation. This could lead to a - # mismatch between the pagination headers info and the actual notes - # array returned, but this is really a edge-case. - paginate(raw_notes) - .reject { |n| n.cross_reference_not_visible_for?(current_user) } + + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + notes = paginate(raw_notes) + notes = prepare_notes_for_rendering(notes) + notes = notes.reject { |n| n.cross_reference_not_visible_for?(current_user) } present notes, with: Entities::Note end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 667bf1ec801..9e888368e7b 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -4,7 +4,7 @@ module API class Pipelines < Grape::API include PaginationParams - before { authenticate! } + before { authenticate_non_get! } params do requires :id, type: String, desc: 'The project ID' @@ -32,6 +32,7 @@ module API end get ':id/pipelines' do authorize! :read_pipeline, user_project + authorize! :read_build, user_project pipelines = PipelinesFinder.new(user_project, current_user, params).execute present paginate(pipelines), with: Entities::PipelineBasic diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index dcc8d94fb79..4f093e9be08 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -65,7 +65,7 @@ module API use :create_params_ee end post ':id/clusters/user' do - authorize! :add_cluster, user_project, 'Instance does not support multiple Kubernetes clusters' + authorize! :add_cluster, user_project user_cluster = ::Clusters::CreateService .new(current_user, create_cluster_user_params) diff --git a/lib/api/container_registry.rb b/lib/api/project_container_repositories.rb index 7dad20a822a..6d53abcc500 100644 --- a/lib/api/container_registry.rb +++ b/lib/api/project_container_repositories.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true module API - class ContainerRegistry < Grape::API + class ProjectContainerRepositories < Grape::API include PaginationParams - REGISTRY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( + REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( tag_name: API::NO_SLASH_URL_PART_REGEX) before { error!('404 Not Found', 404) unless Feature.enabled?(:container_registry_api, user_project, default_enabled: true) } @@ -20,11 +20,14 @@ module API end params do use :pagination + optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included' end get ':id/registry/repositories' do - repositories = user_project.container_repositories.ordered + repositories = ContainerRepositoriesFinder.new( + id: user_project.id, container_type: :project + ).execute - present paginate(repositories), with: Entities::ContainerRegistry::Repository + present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags] end desc 'Delete repository' do @@ -33,7 +36,7 @@ module API params do requires :repository_id, type: Integer, desc: 'The ID of the repository' end - delete ':id/registry/repositories/:repository_id', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + delete ':id/registry/repositories/:repository_id', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_admin_container_image! DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) @@ -49,7 +52,7 @@ module API requires :repository_id, type: Integer, desc: 'The ID of the repository' use :pagination end - get ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + get ':id/registry/repositories/:repository_id/tags', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_read_container_image! tags = Kaminari.paginate_array(repository.tags) @@ -65,7 +68,7 @@ module API optional :keep_n, type: Integer, desc: 'Keep n of latest tags with matching name' optional :older_than, type: String, desc: 'Delete older than: 1h, 1d, 1month' end - delete ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + delete ':id/registry/repositories/:repository_id/tags', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_admin_container_image! message = 'This request has already been made. You can run this at most once an hour for a given container repository' @@ -85,7 +88,7 @@ module API requires :repository_id, type: Integer, desc: 'The ID of the repository' requires :tag_name, type: String, desc: 'The name of the tag' end - get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_read_container_image! validate_tag! @@ -99,7 +102,7 @@ module API requires :repository_id, type: Integer, desc: 'The ID of the repository' requires :tag_name, type: String, desc: 'The name of the tag' end - delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_destroy_container_image! validate_tag! diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 71891e43dcc..bb1b037c08f 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -59,6 +59,7 @@ module API } override_params = import_params.delete(:override_params) + filter_attributes_using_license!(override_params) if override_params project = ::Projects::GitlabProjectsImportService.new( current_user, project_params, override_params diff --git a/lib/api/projects.rb b/lib/api/projects.rb index a7d62014509..3073c14b341 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -115,6 +115,22 @@ module API present_projects load_projects end + + desc 'Get projects starred by a user' do + success Entities::BasicProjectDetails + end + params do + requires :user_id, type: String, desc: 'The ID or username of the user' + use :collection_params + use :statistics_params + end + get ":user_id/starred_projects" do + user = find_user(params[:user_id]) + not_found!('User') unless user + + starred_projects = StarredProjectsFinder.new(user, params: project_finder_params, current_user: current_user).execute + present_projects starred_projects + end end resource :projects do @@ -145,6 +161,7 @@ module API post do attrs = declared_params(include_missing: false) attrs = translate_params_for_compatibility(attrs) + filter_attributes_using_license!(attrs) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -179,6 +196,7 @@ module API attrs = declared_params(include_missing: false) attrs = translate_params_for_compatibility(attrs) + filter_attributes_using_license!(attrs) project = ::Projects::CreateService.new(user, attrs).execute if project.saved? @@ -292,7 +310,7 @@ module API authorize! :change_visibility_level, user_project if attrs[:visibility].present? attrs = translate_params_for_compatibility(attrs) - + filter_attributes_using_license!(attrs) verify_update_project_attrs!(user_project, attrs) result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute @@ -356,6 +374,19 @@ module API end end + desc 'Get the users who starred a project' do + success Entities::UserBasic + end + params do + optional :search, type: String, desc: 'Return list of users matching the search criteria' + use :pagination + end + get ':id/starrers' do + starrers = UsersStarProjectsFinder.new(user_project, params, current_user: current_user).execute + + present paginate(starrers), with: Entities::UserStarsProject + end + desc 'Get languages in project repository' get ':id/languages' do ::Projects::RepositoryLanguagesService @@ -458,11 +489,13 @@ module API end params do optional :search, type: String, desc: 'Return list of users matching the search criteria' + optional :skip_users, type: Array[Integer], desc: 'Filter out users with the specified IDs' use :pagination end get ':id/users' do users = DeclarativePolicy.subject_scope { user_project.team.users } users = users.search(params[:search]) if params[:search].present? + users = users.where_not_in(params[:skip_users]) if params[:skip_users].present? present paginate(users), with: Entities::UserBasic end diff --git a/lib/api/releases.rb b/lib/api/releases.rb index fdd8406388e..7a3d804c30c 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -78,7 +78,7 @@ module API requires :tag_name, type: String, desc: 'The name of the tag', as: :tag optional :name, type: String, desc: 'The name of the release' optional :description, type: String, desc: 'Release notes with markdown support' - optional :released_at, type: DateTime, desc: 'The date when the release will be/was ready. Defaults to the current time.' + optional :released_at, type: DateTime, desc: 'The date when the release will be/was ready.' end put ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do authorize_update_release! diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 4275d911708..c36ee5af63f 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -59,7 +59,7 @@ module API optional :grafana_url, type: String, desc: 'Grafana URL' optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' optional :help_page_hide_commercial_content, type: Boolean, desc: 'Hide marketing-related entries from help' - optional :help_page_support_url, type: String, desc: 'Alternate support URL for help page' + optional :help_page_support_url, type: String, desc: 'Alternate support URL for help page and help dropdown' optional :help_page_text, type: String, desc: 'Custom text displayed on the help page' optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' @@ -124,6 +124,13 @@ module API optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins' optional :local_markdown_version, type: Integer, desc: "Local markdown version, increase this value when any cached markdown should be invalidated" + optional :allow_local_requests_from_hooks_and_services, type: Boolean, desc: 'Deprecated: Use :allow_local_requests_from_web_hooks_and_services instead. Allow requests to the local network from hooks and services.' # support legacy names, can be removed in v5 + optional :snowplow_enabled, type: Grape::API::Boolean, desc: 'Enable Snowplow tracking' + given snowplow_enabled: ->(val) { val } do + requires :snowplow_collector_hostname, type: String, desc: 'The Snowplow collector hostname' + optional :snowplow_cookie_domain, type: String, desc: 'The Snowplow cookie domain' + optional :snowplow_site_id, type: String, desc: 'The Snowplow site name / application ic' + end ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", @@ -158,6 +165,11 @@ module API attrs[:password_authentication_enabled_for_web] = attrs.delete(:password_authentication_enabled) end + # support legacy names, can be removed in v5 + if attrs.has_key?(:allow_local_requests_from_hooks_and_services) + attrs[:allow_local_requests_from_web_hooks_and_services] = attrs.delete(:allow_local_requests_from_hooks_and_services) + end + attrs = filter_attributes_using_license(attrs) if ApplicationSettings::UpdateService.new(current_settings, current_user, attrs).execute diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 7260ecfb5ee..404675bfaec 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -13,6 +13,13 @@ module API 'issues' => ->(iid) { find_project_issue(iid) } }.freeze + helpers do + # EE::API::Todos would override this method + def find_todos + TodosFinder.new(current_user, params).execute + end + end + params do requires :id, type: String, desc: 'The ID of a project' end @@ -41,10 +48,6 @@ module API resource :todos do helpers do - def find_todos - TodosFinder.new(current_user, params).execute - end - def issuable_and_awardable?(type) obj_type = Object.const_get(type) @@ -107,3 +110,5 @@ module API end end end + +API::Todos.prepend_if_ee('EE::API::Todos') diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 0e829c5699b..eeecc390256 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -112,27 +112,6 @@ module API end end - desc 'Take ownership of trigger' do - success Entities::Trigger - end - params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' - end - post ':id/triggers/:trigger_id/take_ownership' do - authenticate! - authorize! :admin_build, user_project - - trigger = user_project.triggers.find(params.delete(:trigger_id)) - break not_found!('Trigger') unless trigger - - if trigger.update(owner: current_user) - status :ok - present trigger, with: Entities::Trigger, current_user: current_user - else - render_validation_error!(trigger) - end - end - desc 'Delete a trigger' do success Entities::Trigger end diff --git a/lib/api/users.rb b/lib/api/users.rb index 41418aa216c..a4ac5b629b8 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -51,7 +51,7 @@ module API optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' optional :avatar, type: File, desc: 'Avatar image for user' - optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile' + optional :private_profile, type: Boolean, default: false, desc: 'Flag indicating the user has a private profile' all_or_none_of :extern_uid, :provider use :optional_params_ee @@ -148,7 +148,7 @@ module API end desc 'Create a user. Available only for admins.' do - success Entities::UserPublic + success Entities::UserWithAdmin end params do requires :email, type: String, desc: 'The email of the user' @@ -168,7 +168,7 @@ module API user = ::Users::CreateService.new(current_user, params).execute(skip_authorization: true) if user.persisted? - present user, with: Entities::UserPublic, current_user: current_user + present user, with: Entities::UserWithAdmin, current_user: current_user else conflict!('Email has already been taken') if User .by_any_email(user.email.downcase) @@ -183,7 +183,7 @@ module API end desc 'Update a user. Available only for admins.' do - success Entities::UserPublic + success Entities::UserWithAdmin end params do requires :id, type: Integer, desc: 'The ID of the user' @@ -215,7 +215,7 @@ module API result = ::Users::UpdateService.new(current_user, user_params.merge(user: user)).execute if result[:status] == :success - present user, with: Entities::UserPublic, current_user: current_user + present user, with: Entities::UserWithAdmin, current_user: current_user else render_validation_error!(user) end diff --git a/lib/api/validations/types/labels_list.rb b/lib/api/validations/types/labels_list.rb index 47cd83c29cf..60277b99106 100644 --- a/lib/api/validations/types/labels_list.rb +++ b/lib/api/validations/types/labels_list.rb @@ -10,7 +10,7 @@ module API when String value.split(',').map(&:strip) when Array - value.map { |v| v.to_s.split(',').map(&:strip) }.flatten + value.flat_map { |v| v.to_s.split(',').map(&:strip) } when LabelsList value else diff --git a/lib/api/variables.rb b/lib/api/variables.rb index af1d7936556..f022b9e665a 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -7,8 +7,6 @@ module API before { authenticate! } before { authorize! :admin_build, user_project } - helpers Helpers::VariablesHelpers - helpers do def filter_variable_parameters(params) # This method exists so that EE can more easily filter out certain @@ -59,8 +57,7 @@ module API optional :protected, type: Boolean, desc: 'Whether the variable is protected' optional :masked, type: Boolean, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' - - use :optional_params_ee + optional :environment_scope, type: String, desc: 'The environment_scope of the variable' end post ':id/variables' do variable_params = declared_params(include_missing: false) @@ -84,8 +81,7 @@ module API optional :protected, type: Boolean, desc: 'Whether the variable is protected' optional :masked, type: Boolean, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' - - use :optional_params_ee + optional :environment_scope, type: String, desc: 'The environment_scope of the variable' end # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do diff --git a/lib/backup/database.rb b/lib/backup/database.rb index cd8e29d14d3..7e457c4982d 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -23,11 +23,6 @@ module Backup dump_pid = case config["adapter"] - when /^mysql/ then - progress.print "Dumping MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) when "postgresql" then progress.print "Dumping PostgreSQL database #{config['database']} ... " pg_env @@ -57,11 +52,6 @@ module Backup restore_pid = case config["adapter"] - when /^mysql/ then - progress.print "Restoring MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysql', *mysql_args, config['database'], in: decompress_rd) when "postgresql" then progress.print "Restoring PostgreSQL database #{config['database']} ... " pg_env @@ -80,23 +70,6 @@ module Backup protected - def mysql_args - args = { - 'host' => '--host', - 'port' => '--port', - 'socket' => '--socket', - 'username' => '--user', - 'encoding' => '--default-character-set', - # SSL - 'sslkey' => '--ssl-key', - 'sslcert' => '--ssl-cert', - 'sslca' => '--ssl-ca', - 'sslcapath' => '--ssl-capath', - 'sslcipher' => '--ssl-cipher' - } - args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact - end - def pg_env args = { 'username' => 'PGUSER', diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 0224dd8fcd1..52af28ce8ec 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -337,6 +337,24 @@ module Banzai @current_project_namespace_path ||= project&.namespace&.full_path end + def records_per_parent + @_records_per_project ||= {} + + @_records_per_project[object_class.to_s.underscore] ||= begin + hash = Hash.new { |h, k| h[k] = {} } + + parent_per_reference.each do |path, parent| + record_ids = references_per_parent[path] + + parent_records(parent, record_ids).each do |record| + hash[parent][record_identifier(record)] = record + end + end + + hash + end + end + private def full_project_path(namespace, project_ref) diff --git a/lib/banzai/filter/ascii_doc_sanitization_filter.rb b/lib/banzai/filter/ascii_doc_sanitization_filter.rb new file mode 100644 index 00000000000..9105e86ad04 --- /dev/null +++ b/lib/banzai/filter/ascii_doc_sanitization_filter.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # Sanitize HTML produced by AsciiDoc/Asciidoctor. + # + # Extends Banzai::Filter::BaseSanitizationFilter with specific rules. + class AsciiDocSanitizationFilter < Banzai::Filter::BaseSanitizationFilter + # Section anchor link pattern + SECTION_LINK_REF_PATTERN = /\A#{Gitlab::Asciidoc::DEFAULT_ADOC_ATTRS['idprefix']}(:?[[:alnum:]]|-|_)+\z/.freeze + SECTION_HEADINGS = %w(h2 h3 h4 h5 h6).freeze + + # Footnote link patterns + FOOTNOTE_LINK_ID_PATTERNS = { + a: /\A_footnoteref_\d+\z/, + div: /\A_footnotedef_\d+\z/ + }.freeze + + # Classes used by Asciidoctor to style components + ADMONITION_CLASSES = %w(fa icon-note icon-tip icon-warning icon-caution icon-important).freeze + CALLOUT_CLASSES = ['conum'].freeze + CHECKLIST_CLASSES = %w(fa fa-check-square-o fa-square-o).freeze + LIST_CLASSES = %w(checklist none no-bullet unnumbered unstyled).freeze + + ELEMENT_CLASSES_WHITELIST = { + span: %w(big small underline overline line-through).freeze, + div: ['admonitionblock'].freeze, + td: ['icon'].freeze, + i: ADMONITION_CLASSES + CALLOUT_CLASSES + CHECKLIST_CLASSES, + ul: LIST_CLASSES, + ol: LIST_CLASSES, + a: ['anchor'].freeze + }.freeze + + def customize_whitelist(whitelist) + # Allow marks + whitelist[:elements].push('mark') + + # Allow any classes in `span`, `i`, `div`, `td`, `ul`, `ol` and `a` elements + # but then remove any unknown classes + whitelist[:attributes]['span'] = %w(class) + whitelist[:attributes]['div'].push('class') + whitelist[:attributes]['td'] = %w(class) + whitelist[:attributes]['i'] = %w(class) + whitelist[:attributes]['ul'] = %w(class) + whitelist[:attributes]['ol'] = %w(class) + whitelist[:attributes]['a'].push('class') + whitelist[:transformers].push(self.class.remove_element_classes) + + # Allow `id` in heading elements for section anchors + SECTION_HEADINGS.each do |header| + whitelist[:attributes][header] = %w(id) + end + whitelist[:transformers].push(self.class.remove_non_heading_ids) + + # Allow `id` in footnote elements + FOOTNOTE_LINK_ID_PATTERNS.keys.each do |element| + whitelist[:attributes][element.to_s].push('id') + end + whitelist[:transformers].push(self.class.remove_non_footnote_ids) + + whitelist + end + + class << self + def remove_non_footnote_ids + lambda do |env| + node = env[:node] + + return unless (pattern = FOOTNOTE_LINK_ID_PATTERNS[node.name.to_sym]) + return unless node.has_attribute?('id') + + return if node['id'] =~ pattern + + node.remove_attribute('id') + end + end + + def remove_non_heading_ids + lambda do |env| + node = env[:node] + + return unless SECTION_HEADINGS.any?(node.name) + return unless node.has_attribute?('id') + + return if node['id'] =~ SECTION_LINK_REF_PATTERN + + node.remove_attribute('id') + end + end + + def remove_element_classes + lambda do |env| + node = env[:node] + + return unless (classes_whitelist = ELEMENT_CLASSES_WHITELIST[node.name.to_sym]) + return unless node.has_attribute?('class') + + classes = node['class'].strip.split(' ') + allowed_classes = (classes & classes_whitelist) + if allowed_classes.empty? + node.remove_attribute('class') + else + node['class'] = allowed_classes.join(' ') + end + end + end + end + end + end +end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 56214043d87..5f2cbc24c60 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -18,6 +18,7 @@ module Banzai # class AutolinkFilter < HTML::Pipeline::Filter include ActionView::Helpers::TagHelper + include Gitlab::Utils::SanitizeNodeLink # Pattern to match text that should be autolinked. # @@ -72,19 +73,11 @@ module Banzai private - # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme - def contains_unsafe?(scheme) - return false unless scheme - - scheme = scheme.strip.downcase - Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } - end - def autolink_match(match) # start by stripping out dangerous links begin uri = Addressable::URI.parse(match) - return match if contains_unsafe?(uri.scheme) + return match unless safe_protocol?(uri.scheme) rescue Addressable::URI::InvalidURIError return match end diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb new file mode 100644 index 00000000000..2dabca3552d --- /dev/null +++ b/lib/banzai/filter/base_sanitization_filter.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # Sanitize HTML produced by markup languages (Markdown, AsciiDoc...). + # Specific rules are implemented in dedicated filters: + # + # - Banzai::Filter::SanitizationFilter (Markdown) + # - Banzai::Filter::AsciiDocSanitizationFilter (AsciiDoc/Asciidoctor) + # + # Extends HTML::Pipeline::SanitizationFilter with common rules. + class BaseSanitizationFilter < HTML::Pipeline::SanitizationFilter + include Gitlab::Utils::StrongMemoize + extend Gitlab::Utils::SanitizeNodeLink + + UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + + def whitelist + strong_memoize(:whitelist) do + whitelist = super.deep_dup + + # Allow span elements + whitelist[:elements].push('span') + + # Allow data-math-style attribute in order to support LaTeX formatting + whitelist[:attributes]['code'] = %w(data-math-style) + whitelist[:attributes]['pre'] = %w(data-math-style) + + # Allow html5 details/summary elements + whitelist[:elements].push('details') + whitelist[:elements].push('summary') + + # Allow abbr elements with title attribute + whitelist[:elements].push('abbr') + whitelist[:attributes]['abbr'] = %w(title) + + # Disallow `name` attribute globally, allow on `a` + whitelist[:attributes][:all].delete('name') + whitelist[:attributes]['a'].push('name') + + # Allow any protocol in `a` elements + # and then remove links with unsafe protocols + whitelist[:protocols].delete('a') + whitelist[:transformers].push(self.class.method(:remove_unsafe_links)) + + # Remove `rel` attribute from `a` elements + whitelist[:transformers].push(self.class.remove_rel) + + customize_whitelist(whitelist) + end + end + + def customize_whitelist(whitelist) + raise NotImplementedError + end + + class << self + def remove_rel + lambda do |env| + if env[:node_name] == 'a' + env[:node].remove_attribute('rel') + end + end + end + end + end + end +end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index c3e5ac41cb8..e1d7b36b9a2 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -19,12 +19,11 @@ module Banzai end def find_object(project, id) - return unless project.is_a?(Project) + return unless project.is_a?(Project) && project.valid_repo? - if project && project.valid_repo? - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/43894 - Gitlab::GitalyClient.allow_n_plus_1_calls { project.commit(id) } - end + _, record = records_per_parent[project].detect { |k, _v| Gitlab::Git.shas_eql?(k, id) } + + record end def referenced_merge_request_commit_shas @@ -66,6 +65,14 @@ module Banzai private + def record_identifier(record) + record.id + end + + def parent_records(parent, ids) + parent.commits_by(oids: ids.to_a) + end + def noteable context[:noteable] end diff --git a/lib/banzai/filter/inline_embeds_filter.rb b/lib/banzai/filter/inline_embeds_filter.rb index 97394fd8f82..9f1ef0796f0 100644 --- a/lib/banzai/filter/inline_embeds_filter.rb +++ b/lib/banzai/filter/inline_embeds_filter.rb @@ -10,8 +10,6 @@ module Banzai # the link, and insert this node after any html content # surrounding the link. def call - return doc unless Feature.enabled?(:gfm_embedded_metrics, context[:project]) - doc.xpath(xpath_search).each do |node| next unless element = element_to_embed(node) diff --git a/lib/banzai/filter/inline_metrics_filter.rb b/lib/banzai/filter/inline_metrics_filter.rb index 0120cc37d6f..c5a328c21b2 100644 --- a/lib/banzai/filter/inline_metrics_filter.rb +++ b/lib/banzai/filter/inline_metrics_filter.rb @@ -15,17 +15,6 @@ module Banzai ) end - # Endpoint FE should hit to collect the appropriate - # chart information - def metrics_dashboard_url(params) - Gitlab::Metrics::Dashboard::Url.build_dashboard_url( - params['namespace'], - params['project'], - params['environment'], - embedded: true - ) - end - # Search params for selecting metrics links. A few # simple checks is enough to boost performance without # the cost of doing a full regex match. @@ -38,6 +27,28 @@ module Banzai def link_pattern Gitlab::Metrics::Dashboard::Url.regex end + + private + + # Endpoint FE should hit to collect the appropriate + # chart information + def metrics_dashboard_url(params) + Gitlab::Metrics::Dashboard::Url.build_dashboard_url( + params['namespace'], + params['project'], + params['environment'], + embedded: true, + **query_params(params['url']) + ) + end + + # Parses query params out from full url string into hash. + # + # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group' + # --> { title: 'Title', group: 'Group' } + def query_params(url) + Gitlab::Metrics::Dashboard::Url.parse_query(url) + end end end end diff --git a/lib/banzai/filter/inline_metrics_redactor_filter.rb b/lib/banzai/filter/inline_metrics_redactor_filter.rb index ff91be2cbb7..4d8a5028898 100644 --- a/lib/banzai/filter/inline_metrics_redactor_filter.rb +++ b/lib/banzai/filter/inline_metrics_redactor_filter.rb @@ -13,8 +13,6 @@ module Banzai # uses to identify the embedded content, removing # only unnecessary nodes. def call - return doc unless Feature.enabled?(:gfm_embedded_metrics, context[:project]) - nodes.each do |node| path = paths_by_node[node] user_has_access = user_access_by_path[path] diff --git a/lib/banzai/filter/issuable_reference_filter.rb b/lib/banzai/filter/issuable_reference_filter.rb index 2963cba91e8..b91ba9f7256 100644 --- a/lib/banzai/filter/issuable_reference_filter.rb +++ b/lib/banzai/filter/issuable_reference_filter.rb @@ -3,22 +3,8 @@ module Banzai module Filter class IssuableReferenceFilter < AbstractReferenceFilter - def records_per_parent - @records_per_project ||= {} - - @records_per_project[object_class.to_s.underscore] ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - parent_per_reference.each do |path, parent| - record_ids = references_per_parent[path] - - parent_records(parent, record_ids).each do |record| - hash[parent][record.iid.to_i] = record - end - end - - hash - end + def record_identifier(record) + record.iid.to_i end def find_object(parent, iid) diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/reference_redactor_filter.rb index 1f091f594f8..485d3fd5fc7 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/reference_redactor_filter.rb @@ -7,12 +7,12 @@ module Banzai # # Expected to be run in its own post-processing pipeline. # - class RedactorFilter < HTML::Pipeline::Filter + class ReferenceRedactorFilter < HTML::Pipeline::Filter def call unless context[:skip_redaction] context = RenderContext.new(project, current_user) - Redactor.new(context).redact([doc]) + ReferenceRedactor.new(context).redact([doc]) end doc diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index a4a06eae7b7..f57e57890f8 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -2,23 +2,13 @@ module Banzai module Filter - # Sanitize HTML + # Sanitize HTML produced by Markdown. # - # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist. - class SanitizationFilter < HTML::Pipeline::SanitizationFilter - include Gitlab::Utils::StrongMemoize - - UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + # Extends Banzai::Filter::BaseSanitizationFilter with specific rules. + class SanitizationFilter < Banzai::Filter::BaseSanitizationFilter + # Styles used by Markdown for table alignment TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/.freeze - def whitelist - strong_memoize(:whitelist) do - customize_whitelist(super.deep_dup) - end - end - - private - def customize_whitelist(whitelist) # Allow table alignment; we whitelist specific text-align values in a # transformer below @@ -26,36 +16,9 @@ module Banzai whitelist[:attributes]['td'] = %w(style) whitelist[:css] = { properties: ['text-align'] } - # Allow span elements - whitelist[:elements].push('span') - - # Allow data-math-style attribute in order to support LaTeX formatting - whitelist[:attributes]['code'] = %w(data-math-style) - whitelist[:attributes]['pre'] = %w(data-math-style) - - # Allow html5 details/summary elements - whitelist[:elements].push('details') - whitelist[:elements].push('summary') - - # Allow abbr elements with title attribute - whitelist[:elements].push('abbr') - whitelist[:attributes]['abbr'] = %w(title) - # Allow the 'data-sourcepos' from CommonMark on all elements whitelist[:attributes][:all].push('data-sourcepos') - # Disallow `name` attribute globally, allow on `a` - whitelist[:attributes][:all].delete('name') - whitelist[:attributes]['a'].push('name') - - # Allow any protocol in `a` elements - # and then remove links with unsafe protocols - whitelist[:protocols].delete('a') - whitelist[:transformers].push(self.class.remove_unsafe_links) - - # Remove `rel` attribute from `a` elements - whitelist[:transformers].push(self.class.remove_rel) - # Remove any `style` properties not required for table alignment whitelist[:transformers].push(self.class.remove_unsafe_table_style) @@ -69,43 +32,6 @@ module Banzai end class << self - def remove_unsafe_links - lambda do |env| - node = env[:node] - - return unless node.name == 'a' - return unless node.has_attribute?('href') - - begin - node['href'] = node['href'].strip - uri = Addressable::URI.parse(node['href']) - - return unless uri.scheme - - # Remove all invalid scheme characters before checking against the - # list of unsafe protocols. - # - # See https://tools.ietf.org/html/rfc3986#section-3.1 - scheme = uri.scheme - .strip - .downcase - .gsub(/[^A-Za-z0-9\+\.\-]+/, '') - - node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(scheme) - rescue Addressable::URI::InvalidURIError - node.remove_attribute('href') - end - end - end - - def remove_rel - lambda do |env| - if env[:node_name] == 'a' - env[:node].remove_attribute('rel') - end - end - end - def remove_unsafe_table_style lambda do |env| node = env[:node] diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 1728a442533..18947679b69 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -8,15 +8,19 @@ module Banzai # Context options: # :project_wiki class WikiLinkFilter < HTML::Pipeline::Filter + include Gitlab::Utils::SanitizeNodeLink + def call return doc unless project_wiki? - doc.search('a:not(.gfm)').each { |el| process_link_attr(el.attribute('href')) } - doc.search('video').each { |el| process_link_attr(el.attribute('src')) } + doc.search('a:not(.gfm)').each { |el| process_link(el.attribute('href'), el) } + + doc.search('video').each { |el| process_link(el.attribute('src'), el) } + doc.search('img').each do |el| attr = el.attribute('data-src') || el.attribute('src') - process_link_attr(attr) + process_link(attr, el) end doc @@ -24,6 +28,11 @@ module Banzai protected + def process_link(link_attr, node) + process_link_attr(link_attr) + remove_unsafe_links({ node: node }, remove_invalid_links: false) + end + def project_wiki? !context[:project_wiki].nil? end diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index 77b5053f38c..f4cc8beeb52 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -4,8 +4,6 @@ module Banzai module Filter class WikiLinkFilter < HTML::Pipeline::Filter class Rewriter - UNSAFE_SLUG_REGEXES = [/\Ajavascript:/i].freeze - def initialize(link_string, wiki:, slug:) @uri = Addressable::URI.parse(link_string) @wiki_base_path = wiki && wiki.wiki_base_path @@ -37,8 +35,6 @@ module Banzai # Of the form `./link`, `../link`, or similar def apply_hierarchical_link_rules! - return if slug_considered_unsafe? - @uri = Addressable::URI.join(@slug, @uri) if @uri.to_s[0] == '.' end @@ -58,10 +54,6 @@ module Banzai def repository_upload? @uri.relative? && @uri.path.starts_with?(Wikis::CreateAttachmentService::ATTACHMENT_PATH) end - - def slug_considered_unsafe? - UNSAFE_SLUG_REGEXES.any? { |r| r.match?(@slug) } - end end end end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 75661ffa233..d6d29f4bfab 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -72,7 +72,7 @@ module Banzai # # Returns an Array containing the redacted documents. def redact_documents(documents) - redactor = Redactor.new(context) + redactor = ReferenceRedactor.new(context) redactor.redact(documents) end diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb index 6be489c6572..d25b74b23b2 100644 --- a/lib/banzai/pipeline/ascii_doc_pipeline.rb +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -5,7 +5,7 @@ module Banzai class AsciiDocPipeline < BasePipeline def self.filters FilterArray[ - Filter::SanitizationFilter, + Filter::AsciiDocSanitizationFilter, Filter::SyntaxHighlightFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index 5c199453638..54af26b41be 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -12,7 +12,7 @@ module Banzai def self.internal_link_filters [ - Filter::RedactorFilter, + Filter::ReferenceRedactorFilter, Filter::InlineMetricsRedactorFilter, Filter::RelativeLinkFilter, Filter::IssuableStateFilter, diff --git a/lib/banzai/redactor.rb b/lib/banzai/reference_redactor.rb index c2da7fec7cc..936436982e7 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/reference_redactor.rb @@ -3,7 +3,7 @@ module Banzai # Class for removing Markdown references a certain user is not allowed to # view. - class Redactor + class ReferenceRedactor attr_reader :context # context - An instance of `Banzai::RenderContext`. @@ -33,7 +33,7 @@ module Banzai # # data - An Array of a Hashes mapping an HTML document to nodes to redact. def redact_document_nodes(all_document_nodes) - all_nodes = all_document_nodes.map { |x| x[:nodes] }.flatten + all_nodes = all_document_nodes.flat_map { |x| x[:nodes] } visible = nodes_visible_to_user(all_nodes) metadata = [] diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index c7239a5eaa6..3cb9ec21e8f 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -55,11 +55,16 @@ module Banzai # Perform multiple render from an Array of Markdown String into an # Array of HTML-safe String of HTML. # - # As the rendered Markdown String can be already cached read all the data - # from the cache using Rails.cache.read_multi operation. If the Markdown String - # is not in the cache or it's not cacheable (no cache_key entry is provided in - # the context) the Markdown String is rendered and stored in the cache so the - # next render call gets the rendered HTML-safe String from the cache. + # The redis cache is completely obviated if we receive a `:rendered` key in the + # context, as it is assumed the item has been pre-rendered somewhere else and there + # is no need to cache it. + # + # If no `:rendered` key is present in the context, as the rendered Markdown String + # can be already cached, read all the data from the cache using + # Rails.cache.read_multi operation. If the Markdown String is not in the cache + # or it's not cacheable (no cache_key entry is provided in the context) the + # Markdown String is rendered and stored in the cache so the next render call + # gets the rendered HTML-safe String from the cache. # # For further explanation see #render method comments. # @@ -76,19 +81,34 @@ module Banzai # => [{ text: '### Hello', # context: { cache_key: [note, :note] } }] def self.cache_collection_render(texts_and_contexts) - items_collection = texts_and_contexts.each_with_index do |item, index| + items_collection = texts_and_contexts.each do |item| context = item[:context] - cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) - item[:cache_key] = cache_key if cache_key + if context.key?(:rendered) + item[:rendered] = context.delete(:rendered) + else + # If the attribute didn't come in pre-rendered, let's prepare it for caching it in redis + cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) + item[:cache_key] = cache_key if cache_key + end end - cacheable_items, non_cacheable_items = items_collection.partition { |item| item.key?(:cache_key) } + cacheable_items, non_cacheable_items = items_collection.group_by do |item| + if item.key?(:rendered) + # We're not really doing anything here as these don't need any processing, but leaving it just in case + # as they could have a cache_key and we don't want them to be re-rendered + :rendered + elsif item.key?(:cache_key) + :cacheable + else + :non_cacheable + end + end.values_at(:cacheable, :non_cacheable) items_in_cache = [] items_not_in_cache = [] - unless cacheable_items.empty? + if cacheable_items.present? items_in_cache = Rails.cache.read_multi(*cacheable_items.map { |item| item[:cache_key] }) items_not_in_cache = cacheable_items.reject do |item| item[:rendered] = items_in_cache[item[:cache_key]] @@ -96,7 +116,7 @@ module Banzai end end - (items_not_in_cache + non_cacheable_items).each do |item| + (items_not_in_cache + Array.wrap(non_cacheable_items)).each do |item| item[:rendered] = render(item[:text], item[:context]) Rails.cache.write(item[:cache_key], item[:rendered]) if item[:cache_key] end @@ -114,7 +134,7 @@ module Banzai # # This method is used to perform state-dependent changes to a String of # HTML, such as removing references that the current user doesn't have - # permission to make (`RedactorFilter`). + # permission to make (`ReferenceRedactorFilter`). # # html - String to process # context - Hash of options to customize output diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb index 1b8dc27793a..127598a76ab 100644 --- a/lib/bitbucket/representation/comment.rb +++ b/lib/bitbucket/representation/comment.rb @@ -4,7 +4,7 @@ module Bitbucket module Representation class Comment < Representation::Base def author - user['username'] + user['nickname'] end def note diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb index a88797cdab9..3f6db9cb75b 100644 --- a/lib/bitbucket/representation/issue.rb +++ b/lib/bitbucket/representation/issue.rb @@ -14,7 +14,7 @@ module Bitbucket end def author - raw.dig('reporter', 'username') + raw.dig('reporter', 'nickname') end def description diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb index 6a0e8b354bf..a498c9bc213 100644 --- a/lib/bitbucket/representation/pull_request.rb +++ b/lib/bitbucket/representation/pull_request.rb @@ -4,7 +4,7 @@ module Bitbucket module Representation class PullRequest < Representation::Base def author - raw.fetch('author', {}).fetch('username', nil) + raw.fetch('author', {}).fetch('nickname', nil) end def description diff --git a/lib/bitbucket_server/client.rb b/lib/bitbucket_server/client.rb index 6a608058813..cf55c692271 100644 --- a/lib/bitbucket_server/client.rb +++ b/lib/bitbucket_server/client.rb @@ -23,8 +23,9 @@ module BitbucketServer BitbucketServer::Representation::Repo.new(parsed_response) end - def repos(page_offset: 0, limit: nil) + def repos(page_offset: 0, limit: nil, filter: nil) path = "/repos" + path += "?name=#{filter}" if filter get_collection(path, :repo, page_offset: page_offset, limit: limit) end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index c80f49f5ae0..82810ea4076 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -7,7 +7,9 @@ module ContainerRegistry class Client attr_accessor :uri - MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json'.freeze + DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json' + OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json' + ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze # Taken from: FaradayMiddleware::FollowRedirects REDIRECT_CODES = Set.new [301, 302, 303, 307] @@ -60,12 +62,13 @@ module ContainerRegistry end def accept_manifest(conn) - conn.headers['Accept'] = MANIFEST_VERSION + conn.headers['Accept'] = ACCEPTED_TYPES conn.response :json, content_type: 'application/json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' - conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' + conn.response :json, content_type: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE + conn.response :json, content_type: OCI_MANIFEST_V1_TYPE end def response_body(response, allow_redirect: false) @@ -79,7 +82,10 @@ module ContainerRegistry def redirect_response(location) return unless location - faraday_redirect.get(location) + uri = URI(@base_uri).merge(location) + raise ArgumentError, "Invalid scheme for #{location}" unless %w[http https].include?(uri.scheme) + + faraday_redirect.get(uri) end def faraday diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index ef41dc560c9..ebea84fa1ca 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -6,6 +6,9 @@ module ContainerRegistry attr_reader :repository, :name + # https://github.com/docker/distribution/commit/3150937b9f2b1b5b096b2634d0e7c44d4a0f89fb + TAG_NAME_REGEX = /^[\w][\w.-]{0,127}$/.freeze + delegate :registry, :client, to: :repository delegate :revision, :short_revision, to: :config_blob, allow_nil: true @@ -13,6 +16,10 @@ module ContainerRegistry @repository, @name = repository, name end + def valid_name? + !name.match(TAG_NAME_REGEX).nil? + end + def valid? manifest.present? end diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb index c83cec9dc4a..45af30f46dc 100644 --- a/lib/expand_variables.rb +++ b/lib/expand_variables.rb @@ -3,6 +3,20 @@ module ExpandVariables class << self def expand(value, variables) + variables_hash = nil + + value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do + variables_hash ||= transform_variables(variables) + variables_hash[$1 || $2] + end + end + + private + + def transform_variables(variables) + # Lazily initialise variables + variables = variables.call if variables.is_a?(Proc) + # Convert hash array to variables if variables.is_a?(Array) variables = variables.reduce({}) do |hash, variable| @@ -11,9 +25,7 @@ module ExpandVariables end end - value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do - variables[$1 || $2] - end + variables end end end diff --git a/lib/feature.rb b/lib/feature.rb index e28333aa58e..c70a6980f19 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -80,6 +80,13 @@ class Feature get(key).disable_group(group) end + def remove(key) + feature = get(key) + return unless persisted?(feature) + + feature.remove + end + def flipper if Gitlab::SafeRequestStore.active? Gitlab::SafeRequestStore[:flipper] ||= build_flipper_instance diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index 67c0b902c0c..656becbffd3 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -5,16 +5,15 @@ require 'set' class Feature class Gitaly # Server feature flags should use '_' to separate words. - # CATFILE_CACHE sets an incorrect example - CATFILE_CACHE = 'catfile-cache'.freeze - SERVER_FEATURE_FLAGS = - [ - CATFILE_CACHE, - 'get_commit_signatures'.freeze + %w[ + get_commit_signatures + cache_invalidator + inforef_uploadpack_cache + get_all_lfs_pointers_go ].freeze - DEFAULT_ON_FLAGS = Set.new([CATFILE_CACHE]).freeze + DEFAULT_ON_FLAGS = Set.new([]).freeze class << self def enabled?(feature_flag) diff --git a/lib/forever.rb b/lib/forever.rb index 0a37118fe68..3f923557441 100644 --- a/lib/forever.rb +++ b/lib/forever.rb @@ -1,15 +1,9 @@ # frozen_string_literal: true class Forever - POSTGRESQL_DATE = DateTime.new(3000, 1, 1) - MYSQL_DATE = DateTime.new(2038, 01, 19) + DATE = DateTime.new(3000, 1, 1) - # MySQL timestamp has a range of '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07' UTC def self.date - if Gitlab::Database.postgresql? - POSTGRESQL_DATE - else - MYSQL_DATE - end + DATE end end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index c62d1071dba..d9d8dcf7900 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_dependency File.expand_path('gitlab/popen', __dir__) +require 'pathname' module Gitlab def self.root @@ -61,7 +61,7 @@ module Gitlab def self.ee? @is_ee ||= - if ENV['IS_GITLAB_EE'].present? + if ENV['IS_GITLAB_EE'] && !ENV['IS_GITLAB_EE'].empty? Gitlab::Utils.to_boolean(ENV['IS_GITLAB_EE']) else # We may use this method when the Rails environment is not loaded. This diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 6eb08f674c2..7ef9f7ef630 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -29,6 +29,10 @@ module Gitlab MAINTAINER_PROJECT_ACCESS = 1 DEVELOPER_MAINTAINER_PROJECT_ACCESS = 2 + # Default subgroup creation level + OWNER_SUBGROUP_ACCESS = 0 + MAINTAINER_SUBGROUP_ACCESS = 1 + class << self delegate :values, to: :options @@ -106,6 +110,13 @@ module Gitlab def project_creation_level_name(name) project_creation_options.key(name) end + + def subgroup_creation_options + { + s_('SubgroupCreationlevel|Owners') => OWNER_SUBGROUP_ACCESS, + s_('SubgroupCreationlevel|Maintainers') => MAINTAINER_SUBGROUP_ACCESS + } + end end def human_access diff --git a/lib/gitlab/action_rate_limiter.rb b/lib/gitlab/action_rate_limiter.rb index c442211e073..0e8707af631 100644 --- a/lib/gitlab/action_rate_limiter.rb +++ b/lib/gitlab/action_rate_limiter.rb @@ -33,16 +33,48 @@ module Gitlab # Increments the given key and returns true if the action should # be throttled. # - # key - An array of ActiveRecord instances - # threshold_value - The maximum number of times this action should occur in the given time interval + # key - An array of ActiveRecord instances or strings + # threshold_value - The maximum number of times this action should occur in the given time interval. If number is zero is considered disabled. def throttled?(key, threshold_value) - self.increment(key) > threshold_value + threshold_value > 0 && + self.increment(key) > threshold_value + end + + # Logs request into auth.log + # + # request - Web request to be logged + # type - A symbol key that represents the request. + # current_user - Current user of the request, it can be nil. + def log_request(request, type, current_user) + request_information = { + message: 'Action_Rate_Limiter_Request', + env: type, + remote_ip: request.ip, + request_method: request.request_method, + path: request.fullpath + } + + if current_user + request_information.merge!({ + user_id: current_user.id, + username: current_user.username + }) + end + + Gitlab::AuthLogger.error(request_information) end private def action_key(key) - serialized = key.map { |obj| "#{obj.class.model_name.to_s.underscore}:#{obj.id}" }.join(":") + serialized = key.map do |obj| + if obj.is_a?(String) + "#{obj}" + else + "#{obj.class.model_name.to_s.underscore}:#{obj.id}" + end + end.join(":") + "action_rate_limiter:#{action}:#{serialized}" end end diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb new file mode 100644 index 00000000000..286c393005f --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +# This module represents the default Cycle Analytics stages that are currently provided by CE +# Each method returns a hash that can be used to build a new stage object. +# +# Example: +# +# params = Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_issue_stage +# Analytics::CycleAnalytics::ProjectStage.new(params) +module Gitlab + module Analytics + module CycleAnalytics + module DefaultStages + def self.all + [ + params_for_issue_stage, + params_for_plan_stage, + params_for_code_stage, + params_for_test_stage, + params_for_review_stage, + params_for_staging_stage, + params_for_production_stage + ] + end + + def self.params_for_issue_stage + { + name: 'issue', + custom: false, # this stage won't be customizable, we provide it as it is + relative_position: 1, # when opening the CycleAnalytics page in CE, this stage will be the first item + start_event_identifier: :issue_created, # IssueCreated class is used as start event + end_event_identifier: :issue_stage_end # IssueStageEnd class is used as end event + } + end + + def self.params_for_plan_stage + { + name: 'plan', + custom: false, + relative_position: 2, + start_event_identifier: :plan_stage_start, + end_event_identifier: :issue_first_mentioned_in_commit + } + end + + def self.params_for_code_stage + { + name: 'code', + custom: false, + relative_position: 3, + start_event_identifier: :code_stage_start, + end_event_identifier: :merge_request_created + } + end + + def self.params_for_test_stage + { + name: 'test', + custom: false, + relative_position: 4, + start_event_identifier: :merge_request_last_build_started, + end_event_identifier: :merge_request_last_build_finished + } + end + + def self.params_for_review_stage + { + name: 'review', + custom: false, + relative_position: 5, + start_event_identifier: :merge_request_created, + end_event_identifier: :merge_request_merged + } + end + + def self.params_for_staging_stage + { + name: 'staging', + custom: false, + relative_position: 6, + start_event_identifier: :merge_request_merged, + end_event_identifier: :merge_request_first_deployed_to_production + } + end + + def self.params_for_production_stage + { + name: 'production', + custom: false, + relative_position: 7, + start_event_identifier: :merge_request_merged, + end_event_identifier: :merge_request_first_deployed_to_production + } + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb new file mode 100644 index 00000000000..d21f344f483 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + # Convention: + # Issue: < 100 + # MergeRequest: >= 100 && < 1000 + # Custom events for default stages: >= 1000 (legacy) + ENUM_MAPPING = { + StageEvents::IssueCreated => 1, + StageEvents::IssueFirstMentionedInCommit => 2, + StageEvents::MergeRequestCreated => 100, + StageEvents::MergeRequestFirstDeployedToProduction => 101, + StageEvents::MergeRequestLastBuildFinished => 102, + StageEvents::MergeRequestLastBuildStarted => 103, + StageEvents::MergeRequestMerged => 104, + StageEvents::CodeStageStart => 1_000, + StageEvents::IssueStageEnd => 1_001, + StageEvents::PlanStageStart => 1_002 + }.freeze + + EVENTS = ENUM_MAPPING.keys.freeze + + # Defines which start_event and end_event pairs are allowed + PAIRING_RULES = { + StageEvents::PlanStageStart => [ + StageEvents::IssueFirstMentionedInCommit + ], + StageEvents::CodeStageStart => [ + StageEvents::MergeRequestCreated + ], + StageEvents::IssueCreated => [ + StageEvents::IssueStageEnd + ], + StageEvents::MergeRequestCreated => [ + StageEvents::MergeRequestMerged + ], + StageEvents::MergeRequestLastBuildStarted => [ + StageEvents::MergeRequestLastBuildFinished + ], + StageEvents::MergeRequestMerged => [ + StageEvents::MergeRequestFirstDeployedToProduction + ] + }.freeze + + def [](identifier) + events.find { |e| e.identifier.to_s.eql?(identifier.to_s) } || raise(KeyError) + end + + # hash for defining ActiveRecord enum: identifier => number + def to_enum + ENUM_MAPPING.each_with_object({}) { |(k, v), hash| hash[k.identifier] = v } + end + + # will be overridden in EE with custom events + def pairing_rules + PAIRING_RULES + end + + # will be overridden in EE with custom events + def events + EVENTS + end + + module_function :[], :to_enum, :pairing_rules, :events + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb new file mode 100644 index 00000000000..ff9c8a79225 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class CodeStageStart < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Issue first mentioned in a commit") + end + + def self.identifier + :code_stage_start + end + + def object_type + MergeRequest + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb new file mode 100644 index 00000000000..a601c9797f8 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class IssueCreated < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Issue created") + end + + def self.identifier + :issue_created + end + + def object_type + Issue + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb new file mode 100644 index 00000000000..7424043ef7b --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class IssueFirstMentionedInCommit < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Issue first mentioned in a commit") + end + + def self.identifier + :issue_first_mentioned_in_commit + end + + def object_type + Issue + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb new file mode 100644 index 00000000000..ceb229c552f --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class IssueStageEnd < SimpleStageEvent + def self.name + PlanStageStart.name + end + + def self.identifier + :issue_stage_end + end + + def object_type + Issue + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb new file mode 100644 index 00000000000..8be00831b4f --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class MergeRequestCreated < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Merge request created") + end + + def self.identifier + :merge_request_created + end + + def object_type + MergeRequest + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb new file mode 100644 index 00000000000..6d7a2c023ff --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class MergeRequestFirstDeployedToProduction < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Merge request first deployed to production") + end + + def self.identifier + :merge_request_first_deployed_to_production + end + + def object_type + MergeRequest + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb new file mode 100644 index 00000000000..12d82fe2c62 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class MergeRequestLastBuildFinished < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Merge request last build finish time") + end + + def self.identifier + :merge_request_last_build_finished + end + + def object_type + MergeRequest + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb new file mode 100644 index 00000000000..9e749b0fdfa --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class MergeRequestLastBuildStarted < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Merge request last build start time") + end + + def self.identifier + :merge_request_last_build_started + end + + def object_type + MergeRequest + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb new file mode 100644 index 00000000000..bbfb5d12992 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class MergeRequestMerged < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Merge request merged") + end + + def self.identifier + :merge_request_merged + end + + def object_type + MergeRequest + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb new file mode 100644 index 00000000000..803317d8b55 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class PlanStageStart < SimpleStageEvent + def self.name + s_("CycleAnalyticsEvent|Issue first associated with a milestone or issue first added to a board") + end + + def self.identifier + :plan_stage_start + end + + def object_type + Issue + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/simple_stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/simple_stage_event.rb new file mode 100644 index 00000000000..253c489d822 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/simple_stage_event.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + # Represents a simple event that usually refers to one database column and does not require additional user input + class SimpleStageEvent < StageEvent + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb new file mode 100644 index 00000000000..a55eee048c2 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + # Base class for expressing an event that can be used for a stage. + class StageEvent + def initialize(params) + @params = params + end + + def self.name + raise NotImplementedError + end + + def self.identifier + raise NotImplementedError + end + + def object_type + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 00c87cce7b6..da65caa6c9c 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -13,6 +13,7 @@ module Gitlab MAX_INCLUDE_DEPTH = 5 DEFAULT_ADOC_ATTRS = { 'showtitle' => true, + 'sectanchors' => true, 'idprefix' => 'user-content-', 'idseparator' => '-', 'env' => 'gitlab', diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 4317992d933..6769bd95c2b 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -46,7 +46,7 @@ module Gitlab user_with_password_for_git(login, password) || Gitlab::Auth::Result.new - rate_limit!(ip, success: result.success?, login: login) + rate_limit!(ip, success: result.success?, login: login) unless skip_rate_limit?(login: login) Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) return result if result.success? || authenticate_using_internal_or_ldap_password? @@ -94,6 +94,7 @@ module Gitlab end end + # rubocop:disable Gitlab/RailsLogger def rate_limit!(ip, success:, login:) rate_limiter = Gitlab::Auth::IpRateLimiter.new(ip) return unless rate_limiter.enabled? @@ -114,9 +115,14 @@ module Gitlab end end end + # rubocop:enable Gitlab/RailsLogger private + def skip_rate_limit?(login:) + ::Ci::Build::CI_REGISTRY_USER == login + end + def authenticate_using_internal_or_ldap_password? Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::Auth::LDAP::Config.enabled? end @@ -192,12 +198,10 @@ module Gitlab end.uniq end - # rubocop: disable CodeReuse/ActiveRecord def deploy_token_check(login, password) return unless password.present? - token = - DeployToken.active.find_by(token: password) + token = DeployToken.active.find_by_token(password) return unless token && login return if login != token.username @@ -208,7 +212,6 @@ module Gitlab Gitlab::Auth::Result.new(token, token.project, :deploy_token, scopes) end end - # rubocop: enable CodeReuse/ActiveRecord def lfs_token_check(login, encoded_token, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) diff --git a/lib/gitlab/auth/activity.rb b/lib/gitlab/auth/activity.rb index 558628b5422..988ff196193 100644 --- a/lib/gitlab/auth/activity.rb +++ b/lib/gitlab/auth/activity.rb @@ -37,14 +37,17 @@ module Gitlab def user_authenticated! self.class.user_authenticated_counter_increment! + + case @opts[:message] + when :two_factor_authenticated + self.class.user_two_factor_authenticated_counter_increment! + end end def user_session_override! self.class.user_session_override_counter_increment! case @opts[:message] - when :two_factor_authenticated - self.class.user_two_factor_authenticated_counter_increment! when :sessionless_sign_in self.class.user_sessionless_authentication_counter_increment! end diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 15b9d5ad6e9..bcb0ecccdf9 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -55,7 +55,7 @@ module Gitlab response = ldap.get_operation_result unless response.code.zero? - Rails.logger.warn("LDAP search error: #{response.message}") + Rails.logger.warn("LDAP search error: #{response.message}") # rubocop:disable Gitlab/RailsLogger end [] @@ -67,7 +67,7 @@ module Gitlab retries += 1 error_message = connection_error_message(error) - Rails.logger.warn(error_message) + Rails.logger.warn(error_message) # rubocop:disable Gitlab/RailsLogger if retries < MAX_SEARCH_RETRIES renew_connection_adapter diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 47d63eb53cf..354f91306f9 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -240,7 +240,7 @@ module Gitlab begin custom_options[:cert] = OpenSSL::X509::Certificate.new(custom_options[:cert]) rescue OpenSSL::X509::CertificateError => e - Rails.logger.error "LDAP TLS Options 'cert' is invalid for provider #{provider}: #{e.message}" + Rails.logger.error "LDAP TLS Options 'cert' is invalid for provider #{provider}: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end @@ -248,7 +248,7 @@ module Gitlab begin custom_options[:key] = OpenSSL::PKey.read(custom_options[:key]) rescue OpenSSL::PKey::PKeyError => e - Rails.logger.error "LDAP TLS Options 'key' is invalid for provider #{provider}: #{e.message}" + Rails.logger.error "LDAP TLS Options 'key' is invalid for provider #{provider}: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index c1517222956..11a4052a109 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -45,7 +45,7 @@ module Gitlab def self.normalize_dn(dn) ::Gitlab::Auth::LDAP::DN.new(dn).to_normalized_s rescue ::Gitlab::Auth::LDAP::DN::FormatError => e - Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") + Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") # rubocop:disable Gitlab/RailsLogger dn end @@ -57,13 +57,13 @@ module Gitlab def self.normalize_uid(uid) ::Gitlab::Auth::LDAP::DN.normalize_value(uid) rescue ::Gitlab::Auth::LDAP::DN::FormatError => e - Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") + Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") # rubocop:disable Gitlab/RailsLogger uid end def initialize(entry, provider) - Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } + Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } # rubocop:disable Gitlab/RailsLogger @entry = entry @provider = provider end diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 72a187377d0..91b9ddc0d00 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -60,8 +60,7 @@ module Gitlab def get_info(key) value = info[key] - Gitlab::Utils.force_utf8(value) if value - value + value.is_a?(String) ? Gitlab::Utils.force_utf8(value) : value end def username_and_email diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 09d1d79fefc..f121dce4cbb 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -77,7 +77,12 @@ module Gitlab end def bypass_two_factor? - false + providers = Gitlab.config.omniauth.allow_bypass_two_factor + if providers.is_a?(Array) + providers.include?(auth_hash.provider) + else + providers + end end protected diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index a5efe33bdc6..bba7e2cbb3c 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -90,8 +90,8 @@ module Gitlab def find_personal_access_token token = current_request.params[PRIVATE_TOKEN_PARAM].presence || - current_request.env[PRIVATE_TOKEN_HEADER].presence - + current_request.env[PRIVATE_TOKEN_HEADER].presence || + parsed_oauth_token return unless token # Expiration, revocation and scopes are verified in `validate_access_token!` @@ -99,9 +99,12 @@ module Gitlab end def find_oauth_access_token - token = Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) + token = parsed_oauth_token return unless token + # PATs with OAuth headers are not handled by OauthAccessToken + return if matches_personal_access_token_length?(token) + # Expiration, revocation and scopes are verified in `validate_access_token!` oauth_token = OauthAccessToken.by_token(token) raise UnauthorizedError unless oauth_token @@ -110,6 +113,14 @@ module Gitlab oauth_token end + def parsed_oauth_token + Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) + end + + def matches_personal_access_token_length?(token) + token.length == PersonalAccessToken::TOKEN_LENGTH + end + # Check if the request is GET/HEAD, or if CSRF token is valid. def verified_request? Gitlab::RequestForgeryProtection.verified?(current_request.env) diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb index cb2bdea755c..c912628d0fc 100644 --- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb +++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb @@ -9,7 +9,7 @@ module Gitlab end def perform(start_id, stop_id) - Rails.logger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") + Rails.logger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") # rubocop:disable Gitlab/RailsLogger update = ' commits_count = ( diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb index 7ee783b8489..3c26982729d 100644 --- a/lib/gitlab/background_migration/archive_legacy_traces.rb +++ b/lib/gitlab/background_migration/archive_legacy_traces.rb @@ -14,7 +14,7 @@ module Gitlab build.trace.archive! rescue => e - Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" + Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index c8d83cc1803..1d9aa050041 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -40,7 +40,7 @@ module Gitlab end def reload! - @shards = Hash[*Shard.all.map { |shard| [shard.name, shard.id] }.flatten] + @shards = Hash[*Shard.all.flat_map { |shard| [shard.name, shard.id] }] end end diff --git a/lib/gitlab/background_migration/calculate_wiki_sizes.rb b/lib/gitlab/background_migration/calculate_wiki_sizes.rb index 886c41a2b9d..e62f5edd0e7 100644 --- a/lib/gitlab/background_migration/calculate_wiki_sizes.rb +++ b/lib/gitlab/background_migration/calculate_wiki_sizes.rb @@ -10,7 +10,7 @@ module Gitlab .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics| statistics.refresh!(only: [:wiki_size]) rescue => e - Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" + Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb index 0e93b2cb2fa..4016b807f21 100644 --- a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -19,20 +19,13 @@ module Gitlab def perform(start_id, stop_id) PagesDomain.where(id: start_id..stop_id).find_each do |domain| - if Gitlab::Database.mysql? - domain.update_columns( - certificate_valid_not_before: domain.x509&.not_before, - certificate_valid_not_after: domain.x509&.not_after - ) - else - # for some reason activerecord doesn't append timezone, iso8601 forces this - domain.update_columns( - certificate_valid_not_before: domain.x509&.not_before&.iso8601, - certificate_valid_not_after: domain.x509&.not_after&.iso8601 - ) - end + # for some reason activerecord doesn't append timezone, iso8601 forces this + domain.update_columns( + certificate_valid_not_before: domain.x509&.not_before&.iso8601, + certificate_valid_not_after: domain.x509&.not_after&.iso8601 + ) rescue => e - Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" + Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/background_migration/fix_cross_project_label_links.rb b/lib/gitlab/background_migration/fix_cross_project_label_links.rb index bf5d7f5f322..20a98c8e141 100644 --- a/lib/gitlab/background_migration/fix_cross_project_label_links.rb +++ b/lib/gitlab/background_migration/fix_cross_project_label_links.rb @@ -108,7 +108,7 @@ module Gitlab next unless matching_label - Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}" + Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}" # rubocop:disable Gitlab/RailsLogger LabelLink.update(label.label_link_id, label_id: matching_label.id) end end diff --git a/lib/gitlab/background_migration/fix_pages_access_level.rb b/lib/gitlab/background_migration/fix_pages_access_level.rb new file mode 100644 index 00000000000..0d49f3dd8c5 --- /dev/null +++ b/lib/gitlab/background_migration/fix_pages_access_level.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # corrects stored pages access level on db depending on project visibility + class FixPagesAccessLevel + # Copy routable here to avoid relying on application logic + module Routable + def build_full_path + if parent && path + parent.build_full_path + '/' + path + else + path + end + end + end + + # Namespace + class Namespace < ApplicationRecord + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + include Routable + + belongs_to :parent, class_name: "Namespace" + end + + # Project + class Project < ActiveRecord::Base + self.table_name = 'projects' + self.inheritance_column = :_type_disabled + + include Routable + + belongs_to :namespace + alias_method :parent, :namespace + alias_attribute :parent_id, :namespace_id + + PRIVATE = 0 + INTERNAL = 10 + PUBLIC = 20 + + def pages_deployed? + Dir.exist?(public_pages_path) + end + + def public_pages_path + File.join(pages_path, 'public') + end + + def pages_path + # TODO: when we migrate Pages to work with new storage types, change here to use disk_path + File.join(Settings.pages.path, build_full_path) + end + end + + # ProjectFeature + class ProjectFeature < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'project_features' + + belongs_to :project + + PRIVATE = 10 + ENABLED = 20 + PUBLIC = 30 + end + + def perform(start_id, stop_id) + fix_public_access_level(start_id, stop_id) + + make_internal_projects_public(start_id, stop_id) + + fix_private_access_level(start_id, stop_id) + end + + private + + def access_control_is_enabled + @access_control_is_enabled = Gitlab.config.pages.access_control + end + + # Public projects are allowed to have only enabled pages_access_level + # which is equivalent to public + def fix_public_access_level(start_id, stop_id) + project_features(start_id, stop_id, ProjectFeature::PUBLIC, Project::PUBLIC).each_batch do |features| + features.update_all(pages_access_level: ProjectFeature::ENABLED) + end + end + + # If access control is disabled and project has pages deployed + # project will become unavailable when access control will become enabled + # we make these projects public to avoid negative surprise to user + def make_internal_projects_public(start_id, stop_id) + return if access_control_is_enabled + + project_features(start_id, stop_id, ProjectFeature::ENABLED, Project::INTERNAL).find_each do |project_feature| + next unless project_feature.project.pages_deployed? + + project_feature.update(pages_access_level: ProjectFeature::PUBLIC) + end + end + + # Private projects are not allowed to have enabled access level, only `private` and `public` + # If access control is enabled, these projects currently behave as if the have `private` pages_access_level + # if access control is disabled, these projects currently behave as if the have `public` pages_access_level + # so we preserve this behaviour for projects with pages already deployed + # for project without pages we always set `private` access_level + def fix_private_access_level(start_id, stop_id) + project_features(start_id, stop_id, ProjectFeature::ENABLED, Project::PRIVATE).find_each do |project_feature| + if access_control_is_enabled + project_feature.update!(pages_access_level: ProjectFeature::PRIVATE) + else + fixed_access_level = project_feature.project.pages_deployed? ? ProjectFeature::PUBLIC : ProjectFeature::PRIVATE + project_feature.update!(pages_access_level: fixed_access_level) + end + end + end + + def project_features(start_id, stop_id, pages_access_level, project_visibility_level) + ProjectFeature.where(id: start_id..stop_id).joins(:project) + .where(pages_access_level: pages_access_level) + .where(projects: { visibility_level: project_visibility_level }) + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_user_namespace_names.rb b/lib/gitlab/background_migration/fix_user_namespace_names.rb new file mode 100644 index 00000000000..1a207121be0 --- /dev/null +++ b/lib/gitlab/background_migration/fix_user_namespace_names.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration fixes the namespaces.name for all user-namespaces that have names + # that aren't equal to the users name. + # Then it uses the updated names of the namespaces to update the associated routes + # For more info see https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/23272 + class FixUserNamespaceNames + def perform(from_id, to_id) + fix_namespace_names(from_id, to_id) + fix_namespace_route_names(from_id, to_id) + end + + def fix_namespace_names(from_id, to_id) + ActiveRecord::Base.connection.execute <<~UPDATE_NAMESPACES + WITH namespaces_to_update AS ( + SELECT + namespaces.id, + users.name AS correct_name + FROM + namespaces + INNER JOIN users ON namespaces.owner_id = users.id + WHERE + namespaces.type IS NULL + AND namespaces.id BETWEEN #{from_id} AND #{to_id} + AND namespaces.name != users.name + ) + UPDATE + namespaces + SET + name = correct_name + FROM + namespaces_to_update + WHERE + namespaces.id = namespaces_to_update.id + UPDATE_NAMESPACES + end + + def fix_namespace_route_names(from_id, to_id) + ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE + WITH routes_to_update AS ( + SELECT + routes.id, + users.name AS correct_name + FROM + routes + INNER JOIN namespaces ON routes.source_id = namespaces.id + INNER JOIN users ON namespaces.owner_id = users.id + WHERE + namespaces.type IS NULL + AND routes.source_type = 'Namespace' + AND namespaces.id BETWEEN #{from_id} AND #{to_id} + AND (routes.name != users.name OR routes.name IS NULL) + ) + UPDATE + routes + SET + name = correct_name + FROM + routes_to_update + WHERE + routes_to_update.id = routes.id + ROUTES_UPDATE + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_user_project_route_names.rb b/lib/gitlab/background_migration/fix_user_project_route_names.rb new file mode 100644 index 00000000000..b84ff32e712 --- /dev/null +++ b/lib/gitlab/background_migration/fix_user_project_route_names.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration fixes the routes.name for all user-projects that have names + # that don't start with the users name. + # For more info see https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/23272 + class FixUserProjectRouteNames + def perform(from_id, to_id) + ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE + WITH routes_to_update AS ( + SELECT + routes.id, + users.name || ' / ' || projects.name AS correct_name + FROM + routes + INNER JOIN projects ON routes.source_id = projects.id + INNER JOIN namespaces ON projects.namespace_id = namespaces.id + INNER JOIN users ON namespaces.owner_id = users.id + WHERE + routes.source_type = 'Project' + AND routes.id BETWEEN #{from_id} AND #{to_id} + AND namespaces.type IS NULL + AND (routes.name NOT LIKE users.name || '%' OR routes.name IS NULL) + ) + UPDATE + routes + SET + name = routes_to_update.correct_name + FROM + routes_to_update + WHERE + routes_to_update.id = routes.id + ROUTES_UPDATE + end + end + end +end diff --git a/lib/gitlab/background_migration/legacy_upload_mover.rb b/lib/gitlab/background_migration/legacy_upload_mover.rb new file mode 100644 index 00000000000..051c1176edb --- /dev/null +++ b/lib/gitlab/background_migration/legacy_upload_mover.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class takes a legacy upload and migrates it to the correct location + class LegacyUploadMover + include Gitlab::Utils::StrongMemoize + + attr_reader :upload, :project, :note + attr_accessor :logger + + def initialize(upload) + @upload = upload + @note = Note.find_by(id: upload.model_id) + @project = note&.project + @logger = Gitlab::BackgroundMigration::Logger.build + end + + def execute + return unless upload + + if !project + # if we don't have models associated with the upload we can not move it + warn('Deleting upload due to model not found.') + + destroy_legacy_upload + elsif note.is_a?(LegacyDiffNote) + return unless move_legacy_diff_file + + migrate_upload + elsif !legacy_file_exists? + warn('Deleting upload due to file not found.') + destroy_legacy_upload + else + migrate_upload + end + end + + private + + def migrate_upload + return unless copy_upload_to_project + + add_upload_link_to_note_text + destroy_legacy_file + destroy_legacy_upload + end + + # we should proceed and log whenever one upload copy fails, no matter the reasons + # rubocop: disable Lint/RescueException + def copy_upload_to_project + @uploader = FileUploader.copy_to(legacy_file_uploader, project) + + logger.info( + message: 'MigrateLegacyUploads: File copied successfully', + old_path: legacy_file_uploader.file.path, new_path: @uploader.file.path + ) + true + rescue Exception => e + warn( + 'File could not be copied to project uploads', + file_path: legacy_file_uploader.file.path, error: e.message + ) + false + end + # rubocop: enable Lint/RescueException + + def destroy_legacy_upload + if note + note.remove_attachment = true + note.save + end + + if upload.destroy + logger.info(message: 'MigrateLegacyUploads: Upload was destroyed.', upload: upload.inspect) + else + warn('MigrateLegacyUploads: Upload destroy failed.') + end + end + + def destroy_legacy_file + legacy_file_uploader.file.delete + end + + def add_upload_link_to_note_text + new_text = "#{note.note} \n #{@uploader.markdown_link}" + # Bypass validations because old data may have invalid + # noteable values. If we fail hard here, we may kill the + # entire background migration, which affects a range of notes. + note.update_attribute(:note, new_text) + end + + def legacy_file_uploader + strong_memoize(:legacy_file_uploader) do + uploader = upload.build_uploader + uploader.retrieve_from_store!(File.basename(upload.path)) + uploader + end + end + + def legacy_file_exists? + legacy_file_uploader.file.exists? + end + + # we should proceed and log whenever one upload copy fails, no matter the reasons + # rubocop: disable Lint/RescueException + def move_legacy_diff_file + old_path = upload.absolute_path + old_path_sub = '-/system/note/attachment' + + if !File.exist?(old_path) || !old_path.include?(old_path_sub) + log_legacy_diff_note_problem(old_path) + return false + end + + new_path = upload.absolute_path.sub(old_path_sub, '-/system/legacy_diff_note/attachment') + new_dir = File.dirname(new_path) + FileUtils.mkdir_p(new_dir) + + FileUtils.mv(old_path, new_path) + rescue Exception => e + log_legacy_diff_note_problem(old_path, new_path, e) + false + end + + def warn(message, params = {}) + logger.warn( + params.merge(message: "MigrateLegacyUploads: #{message}", upload: upload.inspect) + ) + end + + def log_legacy_diff_note_problem(old_path, new_path = nil, error = nil) + warn('LegacyDiffNote upload could not be moved to a new path', + old_path: old_path, new_path: new_path, error: error&.message + ) + end + # rubocop: enable Lint/RescueException + end + end +end diff --git a/lib/gitlab/background_migration/legacy_uploads_migrator.rb b/lib/gitlab/background_migration/legacy_uploads_migrator.rb new file mode 100644 index 00000000000..a9d38a27e0c --- /dev/null +++ b/lib/gitlab/background_migration/legacy_uploads_migrator.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration takes all legacy uploads (that were uploaded using AttachmentUploader) + # and migrate them to the new (FileUploader) location (=under projects). + # + # We have dependencies (uploaders) in this migration because extracting code would add a lot of complexity + # and possible errors could appear as the logic in the uploaders is not trivial. + # + # This migration will be removed in 13.0 in order to get rid of a migration that depends on + # the application code. + class LegacyUploadsMigrator + include Database::MigrationHelpers + + def perform(start_id, end_id) + Upload.where(id: start_id..end_id, uploader: 'AttachmentUploader').find_each do |upload| + LegacyUploadMover.new(upload).execute + end + end + end + end +end diff --git a/lib/gitlab/background_migration/logger.rb b/lib/gitlab/background_migration/logger.rb new file mode 100644 index 00000000000..4ea89771eff --- /dev/null +++ b/lib/gitlab/background_migration/logger.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Logger that can be used for migrations logging + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'migrations' + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb index 5cd638083b0..4377ec2987c 100644 --- a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb +++ b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb @@ -39,10 +39,10 @@ module Gitlab SELECT project_id, id, - artifacts_expire_at, + artifacts_expire_at #{add_missing_db_timezone}, #{LEGACY_PATH_FILE_LOCATION}, - created_at, - created_at, + created_at #{add_missing_db_timezone}, + created_at #{add_missing_db_timezone}, artifacts_file, artifacts_size, COALESCE(artifacts_file_store, #{FILE_LOCAL_STORE}), @@ -81,10 +81,10 @@ module Gitlab SELECT project_id, id, - artifacts_expire_at, + artifacts_expire_at #{add_missing_db_timezone}, #{LEGACY_PATH_FILE_LOCATION}, - created_at, - created_at, + created_at #{add_missing_db_timezone}, + created_at #{add_missing_db_timezone}, artifacts_metadata, NULL, COALESCE(artifacts_metadata_store, #{FILE_LOCAL_STORE}), @@ -121,6 +121,12 @@ module Gitlab AND artifacts_file <> '' SQL end + + def add_missing_db_timezone + return '' unless Gitlab::Database.postgresql? + + 'at time zone \'UTC\'' + end end end end diff --git a/lib/gitlab/background_migration/migrate_null_private_profile_to_false.rb b/lib/gitlab/background_migration/migrate_null_private_profile_to_false.rb new file mode 100644 index 00000000000..32ed6a2756d --- /dev/null +++ b/lib/gitlab/background_migration/migrate_null_private_profile_to_false.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class is responsible for migrating a range of users with private_profile == NULL to false + class MigrateNullPrivateProfileToFalse + # Temporary AR class for users + class User < ActiveRecord::Base + self.table_name = 'users' + end + + def perform(start_id, stop_id) + User.where(private_profile: nil, id: start_id..stop_id).update_all(private_profile: false) + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_stage_index.rb b/lib/gitlab/background_migration/migrate_stage_index.rb index f921233460d..55608529cee 100644 --- a/lib/gitlab/background_migration/migrate_stage_index.rb +++ b/lib/gitlab/background_migration/migrate_stage_index.rb @@ -13,34 +13,22 @@ module Gitlab private def migrate_stage_index_sql(start_id, stop_id) - if Gitlab::Database.postgresql? - <<~SQL - WITH freqs AS ( - SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds - WHERE stage_id BETWEEN #{start_id} AND #{stop_id} - AND stage_idx IS NOT NULL - GROUP BY stage_id, stage_idx - ), indexes AS ( - SELECT DISTINCT stage_id, first_value(stage_idx) - OVER (PARTITION BY stage_id ORDER BY freq DESC) AS index - FROM freqs - ) + <<~SQL + WITH freqs AS ( + SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds + WHERE stage_id BETWEEN #{start_id} AND #{stop_id} + AND stage_idx IS NOT NULL + GROUP BY stage_id, stage_idx + ), indexes AS ( + SELECT DISTINCT stage_id, first_value(stage_idx) + OVER (PARTITION BY stage_id ORDER BY freq DESC) AS index + FROM freqs + ) - UPDATE ci_stages SET position = indexes.index - FROM indexes WHERE indexes.stage_id = ci_stages.id - AND ci_stages.position IS NULL; - SQL - else - <<~SQL - UPDATE ci_stages - SET position = - (SELECT stage_idx FROM ci_builds - WHERE ci_builds.stage_id = ci_stages.id - GROUP BY ci_builds.stage_idx ORDER BY COUNT(*) DESC LIMIT 1) - WHERE ci_stages.id BETWEEN #{start_id} AND #{stop_id} - AND ci_stages.position IS NULL - SQL - end + UPDATE ci_stages SET position = indexes.index + FROM indexes WHERE indexes.stage_id = ci_stages.id + AND ci_stages.position IS NULL; + SQL end end end diff --git a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb index a4c6540c61b..eb4bc0aaf28 100644 --- a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb +++ b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb @@ -18,6 +18,14 @@ module Gitlab execute("INSERT INTO merge_request_assignees (merge_request_id, user_id) #{select_sql}") end + def perform_all_sync(batch_size:) + MergeRequest.each_batch(of: batch_size) do |batch| + range = batch.pluck('MIN(id)', 'MAX(id)').first + + perform(*range) + end + end + private def merge_request_assignees_not_exists_clause diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 755b5ee725a..d2924d10225 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -42,7 +42,7 @@ module Gitlab #{e.message} #{e.backtrace.join("\n ")} MSG - Rails.logger.error(msg) + Rails.logger.error(msg) # rubocop:disable Gitlab/RailsLogger false end end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index 1924f2ffee2..f5fb33f1660 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -176,23 +176,12 @@ module Gitlab self.table_name = 'projects' def self.find_by_full_path(path) - binary = Gitlab::Database.mysql? ? 'BINARY' : '' - order_sql = "(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" + order_sql = "(CASE WHEN routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" where_full_path_in(path).reorder(order_sql).take end def self.where_full_path_in(path) - cast_lower = Gitlab::Database.postgresql? - - path = connection.quote(path) - - where = - if cast_lower - "(LOWER(routes.path) = LOWER(#{path}))" - else - "(routes.path = #{path})" - end - + where = "(LOWER(routes.path) = LOWER(#{connection.quote(path)}))" joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'").where(where) end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 1ee44a3a5a9..2ac51dd7b55 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -55,7 +55,7 @@ module Gitlab def ensure_temporary_tracking_table_exists table_name = :untracked_files_for_uploads - unless ActiveRecord::Base.connection.data_source_exists?(table_name) + unless ActiveRecord::Base.connection.table_exists?(table_name) UntrackedFile.connection.create_table table_name do |t| t.string :path, limit: 600, null: false t.index :path, unique: true @@ -111,7 +111,7 @@ module Gitlab cmd = %W[#{ionice} -c Idle] + cmd if ionice log_msg = "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" - Rails.logger.info log_msg + Rails.logger.info log_msg # rubocop:disable Gitlab/RailsLogger cmd end @@ -133,12 +133,9 @@ module Gitlab def insert_sql(file_paths) if postgresql_pre_9_5? "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" - elsif postgresql? + else "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\ " ON CONFLICT DO NOTHING;" - else # MySQL - "INSERT IGNORE INTO"\ - " #{table_columns_and_values_for_insert(file_paths)};" end end @@ -150,19 +147,13 @@ module Gitlab "#{UntrackedFile.table_name} (path) VALUES #{values}" end - def postgresql? - strong_memoize(:postgresql) do - Gitlab::Database.postgresql? - end - end - def can_bulk_insert_and_ignore_duplicates? !postgresql_pre_9_5? end def postgresql_pre_9_5? strong_memoize(:postgresql_pre_9_5) do - postgresql? && Gitlab::Database.version.to_f < 9.5 + Gitlab::Database.version.to_f < 9.5 end end diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb index 47579d46c1b..9ef6d8654ae 100644 --- a/lib/gitlab/background_migration/remove_restricted_todos.rb +++ b/lib/gitlab/background_migration/remove_restricted_todos.rb @@ -50,14 +50,7 @@ module Gitlab private def remove_non_members_todos(project_id) - if Gitlab::Database.postgresql? - batch_remove_todos_cte(project_id) - else - unauthorized_project_todos(project_id) - .each_batch(of: 5000) do |batch| - batch.delete_all - end - end + batch_remove_todos_cte(project_id) end def remove_confidential_issue_todos(project_id) @@ -90,13 +83,7 @@ module Gitlab next if target_types.empty? - if Gitlab::Database.postgresql? - batch_remove_todos_cte(project_id, target_types) - else - unauthorized_project_todos(project_id) - .where(target_type: target_types) - .delete_all - end + batch_remove_todos_cte(project_id, target_types) end end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 7f7cc62c8ef..15cccc6f287 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -14,7 +14,7 @@ module Gitlab @ref = ref @job = job - @pipeline = @project.ci_pipelines.latest_successful_for(@ref) + @pipeline = @project.ci_pipelines.latest_successful_for_ref(@ref) end def entity diff --git a/lib/gitlab/batch_pop_queueing.rb b/lib/gitlab/batch_pop_queueing.rb new file mode 100644 index 00000000000..61011abddf5 --- /dev/null +++ b/lib/gitlab/batch_pop_queueing.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + ## + # This class is a queuing system for processing expensive tasks in an atomic manner + # with batch poping to let you optimize the total processing time. + # + # In usual queuing system, the first item started being processed immediately + # and the following items wait until the next items have been popped from the queue. + # On the other hand, this queueing system, the former part is same, however, + # it pops the enqueued items as batch. This is especially useful when you want to + # drop redandant items from the queue in order to process important items only, + # thus it's more efficient than the traditional queueing system. + # + # Caveats: + # - The order of the items are not guaranteed because of `sadd` (Redis Sets). + # + # Example: + # ``` + # class TheWorker + # def perform + # result = Gitlab::BatchPopQueueing.new('feature', 'queue').safe_execute([item]) do |items_in_queue| + # item = extract_the_most_important_item_from(items_in_queue) + # expensive_process(item) + # end + # + # if result[:status] == :finished && result[:new_items].present? + # item = extract_the_most_important_item_from(items_in_queue) + # TheWorker.perform_async(item.id) + # end + # end + # end + # ``` + # + class BatchPopQueueing + attr_reader :namespace, :queue_id + + EXTRA_QUEUE_EXPIRE_WINDOW = 1.hour + MAX_COUNTS_OF_POP_ALL = 1000 + + # Initialize queue + # + # @param [String] namespace The namespace of the exclusive lock and queue key. Typically, it's a feature name. + # @param [String] queue_id The identifier of the queue. + # @return [Boolean] + def initialize(namespace, queue_id) + raise ArgumentError if namespace.empty? || queue_id.empty? + + @namespace, @queue_id = namespace, queue_id + end + + ## + # Execute the given block in an exclusive lock. + # If there is the other thread has already working on the block, + # it enqueues the items without processing the block. + # + # @param [Array<String>] new_items New items to be added to the queue. + # @param [Time] lock_timeout The timeout of the exclusive lock. Generally, this value should be longer than the maximum prosess timing of the given block. + # @return [Hash] + # - status => One of the `:enqueued` or `:finished`. + # - new_items => Newly enqueued items during the given block had been processed. + # + # NOTE: If an exception is raised in the block, the poppped items will not be recovered. + # We should NOT re-enqueue the items in this case because it could end up in an infinite loop. + def safe_execute(new_items, lock_timeout: 10.minutes, &block) + enqueue(new_items, lock_timeout + EXTRA_QUEUE_EXPIRE_WINDOW) + + lease = Gitlab::ExclusiveLease.new(lock_key, timeout: lock_timeout) + + return { status: :enqueued } unless uuid = lease.try_obtain + + begin + all_args = pop_all + + yield all_args if block_given? + + { status: :finished, new_items: peek_all } + ensure + Gitlab::ExclusiveLease.cancel(lock_key, uuid) + end + end + + private + + def lock_key + @lock_key ||= "batch_pop_queueing:lock:#{namespace}:#{queue_id}" + end + + def queue_key + @queue_key ||= "batch_pop_queueing:queue:#{namespace}:#{queue_id}" + end + + def enqueue(items, expire_time) + Gitlab::Redis::Queues.with do |redis| + redis.sadd(queue_key, items) + redis.expire(queue_key, expire_time.to_i) + end + end + + def pop_all + Gitlab::Redis::Queues.with do |redis| + redis.spop(queue_key, MAX_COUNTS_OF_POP_ALL) + end + end + + def peek_all + Gitlab::Redis::Queues.with do |redis| + redis.smembers(queue_key) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 8047ef4fa15..24bc73e0de5 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -262,13 +262,19 @@ module Gitlab def pull_request_comment_attributes(comment) { project: project, - note: comment.note, author_id: gitlab_user_id(project, comment.author), + note: comment_note(comment), created_at: comment.created_at, updated_at: comment.updated_at } end + def comment_note(comment) + author = @formatter.author_line(comment.author) unless find_user_id(comment.author) + + author.to_s + comment.note.to_s + end + def log_error(details) logger.error(log_base_data.merge(details)) end diff --git a/lib/gitlab/blob_helper.rb b/lib/gitlab/blob_helper.rb index d3e15a79a8b..fc579ad8d2a 100644 --- a/lib/gitlab/blob_helper.rb +++ b/lib/gitlab/blob_helper.rb @@ -45,7 +45,7 @@ module Gitlab end def image? - ['.png', '.jpg', '.jpeg', '.gif'].include?(extname.downcase) + ['.png', '.jpg', '.jpeg', '.gif', '.svg'].include?(extname.downcase) end # Internal: Lookup mime type for extension. diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb new file mode 100644 index 00000000000..4f47cdef971 --- /dev/null +++ b/lib/gitlab/chaos.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + # Chaos methods for GitLab. + # See https://docs.gitlab.com/ee/development/chaos_endpoints.html for more details. + class Chaos + # leak_mem will retain the specified amount of memory and sleep. + # On return, the memory will be released. + def self.leak_mem(memory_mb, duration_s) + start_time = Time.now + + retainer = [] + # Add `n` 1mb chunks of memory to the retainer array + memory_mb.times { retainer << "x" * 1.megabyte } + + duration_left = [start_time + duration_s - Time.now, 0].max + Kernel.sleep(duration_left) + end + + # cpu_spin will consume all CPU on a single core for the specified duration + def self.cpu_spin(duration_s) + expected_end_time = Time.now + duration_s + + rand while Time.now < expected_end_time + end + + # db_spin will query the database in a tight loop for the specified duration + def self.db_spin(duration_s, interval_s) + expected_end_time = Time.now + duration_s + + while Time.now < expected_end_time + ActiveRecord::Base.connection.execute("SELECT 1") + + end_interval_time = Time.now + [duration_s, interval_s].min + rand while Time.now < end_interval_time + end + end + + # sleep will sleep for the specified duration + def self.sleep(duration_s) + Kernel.sleep(duration_s) + end + + # Kill will send a SIGKILL signal to the current process + def self.kill + Process.kill("KILL", Process.pid) + end + end +end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index fc3223e7442..b7886114e9c 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -194,16 +194,10 @@ module Gitlab end def handle_new_line - css_classes = [] - - if @sections.any? - css_classes = %w[section line] + sections.map { |section| "s_#{section}" } - end - write_in_tag %{<br/>} - write_raw %{<span class="#{css_classes.join(' ')}"></span>} if css_classes.any? + + close_open_tags if @sections.any? && @lineno_in_section == 0 @lineno_in_section += 1 - open_new_tag end def handle_section(scanner) @@ -224,7 +218,7 @@ module Gitlab return if @sections.include?(section) @sections << section - write_raw %{<div class="js-section-start fa fa-caret-down append-right-8 cursor-pointer" data-timestamp="#{timestamp}" data-section="#{data_section_names}" role="button"></div>} + write_raw %{<div class="js-section-start section-start fa fa-caret-down pr-2 cursor-pointer" data-timestamp="#{timestamp}" data-section="#{data_section_names}" role="button"></div>} @lineno_in_section = 0 end @@ -310,11 +304,24 @@ module Gitlab if @sections.any? css_classes << "section" - css_classes << "js-section-header section-header" if @lineno_in_section == 0 + + css_classes << if @lineno_in_section == 0 + "js-section-header section-header cursor-pointer" + else + "line" + end + css_classes += sections.map { |section| "js-s-#{section}" } end - @out << %{<span class="#{css_classes.join(' ')}">} + close_open_tags + + @out << if css_classes.any? + %{<span class="#{css_classes.join(' ')}">} + else + %{<span>} + end + @n_open_tags += 1 end diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb index 0698136166a..e9c8864123f 100644 --- a/lib/gitlab/ci/build/policy/variables.rb +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -10,7 +10,7 @@ module Gitlab end def satisfied_by?(pipeline, seed) - variables = seed.to_resource.scoped_variables_hash + variables = seed.scoped_variables_hash statements = @expressions.map do |statement| ::Gitlab::Ci::Pipeline::Expression::Statement diff --git a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb index e6e0aaab60b..6ab4fca3854 100644 --- a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb +++ b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb @@ -8,31 +8,51 @@ module Gitlab def unmet? deployment_cluster.present? && deployment_cluster.managed? && - (kubernetes_namespace.new_record? || kubernetes_namespace.service_account_token.blank?) + missing_namespace? end def complete! return unless unmet? - create_or_update_namespace + create_namespace end private + def missing_namespace? + kubernetes_namespace.nil? || kubernetes_namespace.service_account_token.blank? + end + def deployment_cluster build.deployment&.cluster end + def environment + build.deployment.environment + end + def kubernetes_namespace strong_memoize(:kubernetes_namespace) do - deployment_cluster.find_or_initialize_kubernetes_namespace_for_project(build.project) + Clusters::KubernetesNamespaceFinder.new( + deployment_cluster, + project: environment.project, + environment_slug: environment.slug, + allow_blank_token: true + ).execute end end - def create_or_update_namespace + def create_namespace Clusters::Gcp::Kubernetes::CreateOrUpdateNamespaceService.new( cluster: deployment_cluster, - kubernetes_namespace: kubernetes_namespace + kubernetes_namespace: kubernetes_namespace || build_namespace_record + ).execute + end + + def build_namespace_record + Clusters::BuildKubernetesNamespaceService.new( + deployment_cluster, + environment: environment ).execute end end diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb new file mode 100644 index 00000000000..89623a809c9 --- /dev/null +++ b/lib/gitlab/ci/build/rules.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules + include ::Gitlab::Utils::StrongMemoize + + Result = Struct.new(:when, :start_in) + + def initialize(rule_hashes, default_when = 'on_success') + @rule_list = Rule.fabricate_list(rule_hashes) + @default_when = default_when + end + + def evaluate(pipeline, build) + if @rule_list.nil? + Result.new(@default_when) + elsif matched_rule = match_rule(pipeline, build) + Result.new( + matched_rule.attributes[:when] || @default_when, + matched_rule.attributes[:start_in] + ) + else + Result.new('never') + end + end + + private + + def match_rule(pipeline, build) + @rule_list.find { |rule| rule.matches?(pipeline, build) } + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule.rb b/lib/gitlab/ci/build/rules/rule.rb new file mode 100644 index 00000000000..8d52158c8d2 --- /dev/null +++ b/lib/gitlab/ci/build/rules/rule.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules::Rule + attr_accessor :attributes + + def self.fabricate_list(list) + list.map(&method(:new)) if list + end + + def initialize(spec) + @clauses = [] + @attributes = {} + + spec.each do |type, value| + if clause = Clause.fabricate(type, value) + @clauses << clause + else + @attributes.merge!(type => value) + end + end + end + + def matches?(pipeline, build) + @clauses.all? { |clause| clause.satisfied_by?(pipeline, build) } + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause.rb b/lib/gitlab/ci/build/rules/rule/clause.rb new file mode 100644 index 00000000000..ff0baf3348c --- /dev/null +++ b/lib/gitlab/ci/build/rules/rule/clause.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules::Rule::Clause + ## + # Abstract class that defines an interface of a single + # job rule specification. + # + # Used for job's inclusion rules configuration. + # + UnknownClauseError = Class.new(StandardError) + + def self.fabricate(type, value) + type = type.to_s.camelize + + self.const_get(type).new(value) if self.const_defined?(type) + end + + def initialize(spec) + @spec = spec + end + + def satisfied_by?(pipeline, seed = nil) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb new file mode 100644 index 00000000000..81d2ee6c24c --- /dev/null +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules::Rule::Clause::Changes < Rules::Rule::Clause + def initialize(globs) + @globs = Array(globs) + end + + def satisfied_by?(pipeline, seed) + return true if pipeline.modified_paths.nil? + + pipeline.modified_paths.any? do |path| + @globs.any? do |glob| + File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause/if.rb b/lib/gitlab/ci/build/rules/rule/clause/if.rb new file mode 100644 index 00000000000..18c3b450f95 --- /dev/null +++ b/lib/gitlab/ci/build/rules/rule/clause/if.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules::Rule::Clause::If < Rules::Rule::Clause + def initialize(expression) + @expression = expression + end + + def satisfied_by?(pipeline, seed) + variables = seed.scoped_variables_hash + + ::Gitlab::Ci::Pipeline::Expression::Statement.new(@expression, variables).truthful? + end + end + end + end +end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 7cabaadb122..3fbfdffe277 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -21,16 +21,10 @@ module Gitlab module MonthlyInterval # rubocop: disable CodeReuse/ActiveRecord def grouped_count(query) - if Gitlab::Database.postgresql? - query - .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") - .count(:created_at) - .transform_keys(&:squish) - else - query - .group("DATE_FORMAT(#{::Ci::Pipeline.table_name}.created_at, '01 %M %Y')") - .count(:created_at) - end + query + .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") + .count(:created_at) + .transform_keys(&:squish) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 5ab795359b8..6e11c582750 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,17 +11,28 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[tags script only except type image services + ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze + ALLOWED_KEYS = %i[tags script only except rules type image services allow_failure type stage when start_in artifacts cache - dependencies before_script after_script variables + dependencies needs before_script after_script variables environment coverage retry parallel extends].freeze + REQUIRED_BY_NEEDS = %i[stage].freeze + validations do + validates :config, type: Hash validates :config, allowed_keys: ALLOWED_KEYS + validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs? validates :config, presence: true validates :script, presence: true validates :name, presence: true validates :name, type: Symbol + validates :config, + disallowed_keys: { + in: %i[only except when start_in], + message: 'key may not be used with `rules`' + }, + if: :has_rules? with_options allow_nil: true do validates :tags, array_of_strings: true @@ -29,16 +40,29 @@ module Gitlab validates :parallel, numericality: { only_integer: true, greater_than_or_equal_to: 2, less_than_or_equal_to: 50 } - validates :when, - inclusion: { in: %w[on_success on_failure always manual delayed], - message: 'should be on_success, on_failure, ' \ - 'always, manual or delayed' } + validates :when, inclusion: { + in: ALLOWED_WHEN, + message: "should be one of: #{ALLOWED_WHEN.join(', ')}" + } + validates :dependencies, array_of_strings: true + validates :needs, array_of_strings: true validates :extends, array_of_strings_or_string: true + validates :rules, array_of_hashes: true end validates :start_in, duration: { limit: '1 day' }, if: :delayed? - validates :start_in, absence: true, unless: :delayed? + validates :start_in, absence: true, if: -> { has_rules? || !delayed? } + + validate do + next unless dependencies.present? + next unless needs.present? + + missing_needs = dependencies - needs + if missing_needs.any? + errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") + end + end end entry :before_script, Entry::Script, @@ -77,6 +101,9 @@ module Gitlab entry :except, Entry::Policy, description: 'Refs policy this job will be executed for.' + entry :rules, Entry::Rules, + description: 'List of evaluable Rules to determine job inclusion.' + entry :variables, Entry::Variables, description: 'Environment variables available for this job.' @@ -95,10 +122,10 @@ module Gitlab helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, :artifacts, :environment, :coverage, :retry, - :parallel + :parallel, :needs attributes :script, :tags, :allow_failure, :when, :dependencies, - :retry, :parallel, :extends, :start_in + :needs, :retry, :parallel, :extends, :start_in, :rules def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -137,6 +164,10 @@ module Gitlab self.when == 'delayed' end + def has_rules? + @config.try(:key?, :rules) + end + def ignored? allow_failure.nil? ? manual_action? : allow_failure end @@ -178,7 +209,8 @@ module Gitlab parallel: parallel_defined? ? parallel_value.to_i : nil, artifacts: artifacts_value, after_script: after_script_value, - ignore: ignored? } + ignore: ignored?, + needs: needs_defined? ? needs_value : nil } end end end diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb new file mode 100644 index 00000000000..65cad0880f5 --- /dev/null +++ b/lib/gitlab/ci/config/entry/rules.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Rules < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, presence: true + validates :config, type: Array + end + + def compose!(deps = nil) + super(deps) do + @config.each_with_index do |rule, index| + @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Rules::Rule) + .value(rule) + .with(key: "rule", parent: self, description: "rule definition.") # rubocop:disable CodeReuse/ActiveRecord + .create! + end + + @entries.each_value do |entry| + entry.compose!(deps) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb new file mode 100644 index 00000000000..1f2a34ec90e --- /dev/null +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Rules::Rule < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + CLAUSES = %i[if changes].freeze + ALLOWED_KEYS = %i[if changes when start_in].freeze + ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze + + attributes :if, :changes, :when, :start_in + + validations do + validates :config, presence: true + validates :config, type: { with: Hash } + validates :config, allowed_keys: ALLOWED_KEYS + validates :config, disallowed_keys: %i[start_in], unless: :specifies_delay? + validates :start_in, presence: true, if: :specifies_delay? + validates :start_in, duration: { limit: '1 day' }, if: :specifies_delay? + + with_options allow_nil: true do + validates :if, expression: true + validates :changes, array_of_strings: true + validates :when, allowed_values: { in: ALLOWED_WHEN } + end + end + + def specifies_delay? + self.when == 'delayed' + end + + def default + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 191f5d09645..09f9bf5f69f 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -4,61 +4,63 @@ module Gitlab module Ci class Config class Normalizer + include Gitlab::Utils::StrongMemoize + def initialize(jobs_config) @jobs_config = jobs_config end def normalize_jobs - extract_parallelized_jobs! - return @jobs_config if @parallelized_jobs.empty? + return @jobs_config if parallelized_jobs.empty? + + expand_parallelize_jobs do |job_name, config| + if config[:dependencies] + config[:dependencies] = expand_names(config[:dependencies]) + end - parallelized_config = parallelize_jobs - parallelize_dependencies(parallelized_config) + if config[:needs] + config[:needs] = expand_names(config[:needs]) + end + + config + end end private - def extract_parallelized_jobs! - @parallelized_jobs = {} + def expand_names(job_names) + return unless job_names - @jobs_config.each do |job_name, config| - if config[:parallel] - @parallelized_jobs[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) - end + job_names.flat_map do |job_name| + parallelized_jobs[job_name.to_sym] || job_name end - - @parallelized_jobs end - def parallelize_jobs - @jobs_config.each_with_object({}) do |(job_name, config), hash| - if @parallelized_jobs.key?(job_name) - @parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) } - else - hash[job_name] = config - end + def parallelized_jobs + strong_memoize(:parallelized_jobs) do + @jobs_config.each_with_object({}) do |(job_name, config), hash| + next unless config[:parallel] - hash + hash[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) + end end end - def parallelize_dependencies(parallelized_config) - parallelized_job_names = @parallelized_jobs.keys.map(&:to_s) - parallelized_config.each_with_object({}) do |(job_name, config), hash| - if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any? - parallelized_deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten - deps = config[:dependencies] - intersection + parallelized_deps - hash[job_name] = config.merge(dependencies: deps) + def expand_parallelize_jobs + @jobs_config.each_with_object({}) do |(job_name, config), hash| + if parallelized_jobs.key?(job_name) + parallelized_jobs[job_name].each_with_index do |name, index| + hash[name.to_sym] = + yield(name, config.merge(name: name, instance: index + 1)) + end else - hash[job_name] = config + hash[job_name] = yield(job_name, config) end - - hash end end def self.parallelize_job_names(name, total) - Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] } + Array.new(total) { |index| "#{name} #{index + 1}/#{total}" } end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index c911bfa7ff6..afad391e8e0 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -20,6 +20,12 @@ module Gitlab end end + def uses_unsupported_legacy_trigger? + trigger_request.present? && + trigger_request.trigger.legacy? && + !trigger_request.trigger.supports_legacy_tokens? + end + def branch_exists? strong_memoize(:is_branch) do project.repository.branch_exists?(ref) diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 0405292a25b..65029f5ce7f 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -23,12 +23,17 @@ module Gitlab @command.seeds_block&.call(pipeline) ## - # Populate pipeline with all stages, and stages with builds. + # Gather all runtime build/stage errors # - pipeline.stage_seeds.each do |stage| - pipeline.stages << stage.to_resource + if seeds_errors = pipeline.stage_seeds.flat_map(&:errors).compact.presence + return error(seeds_errors.join("\n")) end + ## + # Populate pipeline with all stages, and stages with builds. + # + pipeline.stages = pipeline.stage_seeds.map(&:to_resource) + if pipeline.stages.none? return error('No stages / jobs for this pipeline.') end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index aaa3daddcc5..357a1d55b3b 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -14,6 +14,10 @@ module Gitlab return error('Pipelines are disabled!') end + if @command.uses_unsupported_legacy_trigger? + return error('Trigger token is invalid because is not owned by any user') + end + unless allowed_to_trigger_pipeline? if can?(current_user, :create_pipeline, project) return error("Insufficient permissions for protected ref '#{command.ref}'") diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index 942e4e55323..f7b0720d4a9 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -11,8 +11,9 @@ module Gitlab def evaluate(variables = {}) text = @left.evaluate(variables) regexp = @right.evaluate(variables) + return false unless regexp - regexp.scan(text.to_s).any? + regexp.scan(text.to_s).present? end def self.build(_value, behind, ahead) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb index 831c27fa0ea..02479ed28a4 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb @@ -11,8 +11,9 @@ module Gitlab def evaluate(variables = {}) text = @left.evaluate(variables) regexp = @right.evaluate(variables) + return true unless regexp - regexp.scan(text.to_s).none? + regexp.scan(text.to_s).empty? end def self.build(_value, behind, ahead) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index e4cf360a1c1..0212fa9d661 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -8,11 +8,10 @@ module Gitlab require_dependency 're2' class Pattern < Lexeme::Value - PATTERN = %r{^/.+/[ismU]*$}.freeze - NEW_PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze + PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze def initialize(regexp) - @value = self.class.eager_matching_with_escape_characters? ? regexp.gsub(/\\\//, '/') : regexp + @value = regexp.gsub(/\\\//, '/') unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value) raise Lexer::SyntaxError, 'Invalid regular expression!' @@ -26,16 +25,12 @@ module Gitlab end def self.pattern - eager_matching_with_escape_characters? ? NEW_PATTERN : PATTERN + PATTERN end def self.build(string) new(string) end - - def self.eager_matching_with_escape_characters? - Feature.enabled?(:ci_variables_complex_expressions, default_enabled: true) - end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index 22c210ae26b..7d7582612f9 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -17,17 +17,6 @@ module Gitlab Expression::Lexeme::Equals, Expression::Lexeme::Matches, Expression::Lexeme::NotEquals, - Expression::Lexeme::NotMatches - ].freeze - - NEW_LEXEMES = [ - Expression::Lexeme::Variable, - Expression::Lexeme::String, - Expression::Lexeme::Pattern, - Expression::Lexeme::Null, - Expression::Lexeme::Equals, - Expression::Lexeme::Matches, - Expression::Lexeme::NotEquals, Expression::Lexeme::NotMatches, Expression::Lexeme::And, Expression::Lexeme::Or @@ -58,7 +47,7 @@ module Gitlab return tokens if @scanner.eos? - lexeme = available_lexemes.find do |type| + lexeme = LEXEMES.find do |type| type.scan(@scanner).tap do |token| tokens.push(token) if token.present? end @@ -71,10 +60,6 @@ module Gitlab raise Lexer::SyntaxError, 'Too many tokens!' end - - def available_lexemes - Feature.enabled?(:ci_variables_complex_expressions, default_enabled: true) ? NEW_LEXEMES : LEXEMES - end end end end diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index 589bf32a4d7..edb55edf356 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -13,39 +13,6 @@ module Gitlab end def tree - if Feature.enabled?(:ci_variables_complex_expressions, default_enabled: true) - rpn_parse_tree - else - reverse_descent_parse_tree - end - end - - def self.seed(statement) - new(Expression::Lexer.new(statement).tokens) - end - - private - - # This produces a reverse descent parse tree. - # It does not support precedence of operators. - def reverse_descent_parse_tree - while token = @tokens.next - case token.type - when :operator - token.build(@nodes.pop, tree).tap do |node| - @nodes.push(node) - end - when :value - token.build.tap do |leaf| - @nodes.push(leaf) - end - end - end - rescue StopIteration - @nodes.last || Lexeme::Null.new - end - - def rpn_parse_tree results = [] tokens_rpn.each do |token| @@ -70,6 +37,12 @@ module Gitlab results.pop end + def self.seed(statement) + new(Expression::Lexer.new(statement).tokens) + end + + private + # Parse the expression into Reverse Polish Notation # (See: Shunting-yard algorithm) def tokens_rpn diff --git a/lib/gitlab/ci/pipeline/seed/base.rb b/lib/gitlab/ci/pipeline/seed/base.rb index 1fd3a61017f..e9e22569ae0 100644 --- a/lib/gitlab/ci/pipeline/seed/base.rb +++ b/lib/gitlab/ci/pipeline/seed/base.rb @@ -13,6 +13,10 @@ module Gitlab raise NotImplementedError end + def errors + raise NotImplementedError + end + def to_resource raise NotImplementedError end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index d8296940a04..1066331062b 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -7,39 +7,65 @@ module Gitlab class Build < Seed::Base include Gitlab::Utils::StrongMemoize - delegate :dig, to: :@attributes + delegate :dig, to: :@seed_attributes - def initialize(pipeline, attributes) + # When the `ci_dag_limit_needs` is enabled it uses the lower limit + LOW_NEEDS_LIMIT = 5 + HARD_NEEDS_LIMIT = 50 + + def initialize(pipeline, attributes, previous_stages) @pipeline = pipeline - @attributes = attributes + @seed_attributes = attributes + @previous_stages = previous_stages + @needs_attributes = dig(:needs_attributes) + + @using_rules = attributes.key?(:rules) + @using_only = attributes.key?(:only) + @using_except = attributes.key?(:except) @only = Gitlab::Ci::Build::Policy .fabricate(attributes.delete(:only)) @except = Gitlab::Ci::Build::Policy .fabricate(attributes.delete(:except)) + @rules = Gitlab::Ci::Build::Rules + .new(attributes.delete(:rules)) + end + + def name + dig(:name) end def included? strong_memoize(:inclusion) do - @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } && - @except.none? { |spec| spec.satisfied_by?(@pipeline, self) } + if @using_rules + included_by_rules? + elsif @using_only || @using_except + all_of_only? && none_of_except? + else + true + end + end + end + + def errors + return unless included? + + strong_memoize(:errors) do + needs_errors end end def attributes - @attributes.merge( - pipeline: @pipeline, - project: @pipeline.project, - user: @pipeline.user, - ref: @pipeline.ref, - tag: @pipeline.tag, - trigger_request: @pipeline.legacy_trigger, - protected: @pipeline.protected_ref? - ) + @seed_attributes + .deep_merge(pipeline_attributes) + .deep_merge(rules_attributes) end def bridge? - @attributes.to_h.dig(:options, :trigger).present? + attributes_hash = @seed_attributes.to_h + attributes_hash.dig(:options, :trigger).present? || + (attributes_hash.dig(:options, :bridge_needs).instance_of?(Hash) && + attributes_hash.dig(:options, :bridge_needs, :pipeline).present?) end def to_resource @@ -51,6 +77,77 @@ module Gitlab end end end + + def scoped_variables_hash + strong_memoize(:scoped_variables_hash) do + # This is a temporary piece of technical debt to allow us access + # to the CI variables to evaluate rules before we persist a Build + # with the result. We should refactor away the extra Build.new, + # but be able to get CI Variables directly from the Seed::Build. + ::Ci::Build.new( + @seed_attributes.merge(pipeline_attributes) + ).scoped_variables_hash + end + end + + private + + def all_of_only? + @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } + end + + def none_of_except? + @except.none? { |spec| spec.satisfied_by?(@pipeline, self) } + end + + def needs_errors + return if @needs_attributes.nil? + + if @needs_attributes.size > max_needs_allowed + return [ + "#{name}: one job can only need #{max_needs_allowed} others, but you have listed #{@needs_attributes.size}. " \ + "See needs keyword documentation for more details" + ] + end + + @needs_attributes.flat_map do |need| + result = @previous_stages.any? do |stage| + stage.seeds_names.include?(need[:name]) + end + + "#{name}: needs '#{need[:name]}'" unless result + end.compact + end + + def max_needs_allowed + if Feature.enabled?(:ci_dag_limit_needs, @project, default_enabled: true) + LOW_NEEDS_LIMIT + else + HARD_NEEDS_LIMIT + end + end + + def pipeline_attributes + { + pipeline: @pipeline, + project: @pipeline.project, + user: @pipeline.user, + ref: @pipeline.ref, + tag: @pipeline.tag, + trigger_request: @pipeline.legacy_trigger, + protected: @pipeline.protected_ref? + } + end + + def included_by_rules? + rules_attributes[:when] != 'never' + end + + def rules_attributes + strong_memoize(:rules_attributes) do + @using_rules ? @rules.evaluate(@pipeline, self).to_h.compact : {} + end + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 9c15064756a..b600df2f656 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -10,12 +10,13 @@ module Gitlab delegate :size, to: :seeds delegate :dig, to: :seeds - def initialize(pipeline, attributes) + def initialize(pipeline, attributes, previous_stages) @pipeline = pipeline @attributes = attributes + @previous_stages = previous_stages @builds = attributes.fetch(:builds).map do |attributes| - Seed::Build.new(@pipeline, attributes) + Seed::Build.new(@pipeline, attributes, previous_stages) end end @@ -32,6 +33,18 @@ module Gitlab end end + def errors + strong_memoize(:errors) do + seeds.flat_map(&:errors).compact + end + end + + def seeds_names + strong_memoize(:seeds_names) do + seeds.map(&:name).to_set + end + end + def included? seeds.any? end @@ -39,13 +52,7 @@ module Gitlab def to_resource strong_memoize(:stage) do ::Ci::Stage.new(attributes).tap do |stage| - seeds.each do |seed| - if seed.bridge? - stage.bridges << seed.to_resource - else - stage.builds << seed.to_resource - end - end + stage.statuses = seeds.map(&:to_resource) end end end diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index d01b09f1398..df572188194 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -10,7 +10,7 @@ module Gitlab image: 'illustrations/manual_action.svg', size: 'svg-394', title: _('This job requires a manual action'), - content: _('This job depends on a user to trigger its process. Often they are used to deploy code to production environments') + content: _('This job requires manual intervention to start. Before starting this job, you can add variables below for last-minute configuration changes.') } end diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index 3446644eff8..2a0bf060c9b 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -34,11 +34,9 @@ module Gitlab def extended_statuses return @extended_statuses if defined?(@extended_statuses) - groups = self.class.extended_statuses.map do |group| + @extended_statuses = self.class.extended_statuses.flat_map do |group| Array(group).find { |status| status.matches?(@subject, @user) } - end - - @extended_statuses = groups.flatten.compact + end.compact end def self.extended_statuses diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index cf3d261c1cb..5c1c0c142e5 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -50,13 +50,12 @@ variables: POSTGRES_DB: $CI_ENVIRONMENT_SLUG POSTGRES_VERSION: 9.6.2 - KUBERNETES_VERSION: 1.11.10 - HELM_VERSION: 2.14.0 - DOCKER_DRIVER: overlay2 ROLLOUT_RESOURCE_TYPE: deployment + DOCKER_TLS_CERTDIR: "" # https://gitlab.com/gitlab-org/gitlab-runner/issues/4501 + stages: - build - test diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index a09217e8cf0..b0a79950667 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -2,6 +2,8 @@ performance: stage: performance image: docker:stable allow_failure: true + variables: + DOCKER_TLS_CERTDIR: "" services: - docker:stable-dind script: diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 18f7290e1d9..8061da968ed 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,6 +1,8 @@ build: stage: build image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image/master:stable" + variables: + DOCKER_TLS_CERTDIR: "" services: - docker:stable-dind script: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 8a84744aa2d..3adc6a72874 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -6,6 +6,7 @@ code_quality: - docker:stable-dind variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" script: - | if ! docker info &>/dev/null; then @@ -22,6 +23,7 @@ code_quality: reports: codequality: gl-code-quality-report.json expire_in: 1 week + dependencies: [] only: - branches - tags diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 108f0119ae1..a8ec2d4781d 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,14 +1,17 @@ +.auto-deploy: + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.1.0" + review: + extends: .auto-deploy stage: review script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy persist_environment_url environment: name: review/$CI_COMMIT_REF_NAME url: http://$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -27,13 +30,13 @@ review: - $REVIEW_DISABLED stop_review: + extends: .auto-deploy stage: cleanup variables: GIT_STRATEGY: none script: - - install_dependencies - - initialize_tiller - - delete + - auto-deploy initialize_tiller + - auto-deploy delete environment: name: review/$CI_COMMIT_REF_NAME action: stop @@ -57,15 +60,15 @@ stop_review: # STAGING_ENABLED. staging: + extends: .auto-deploy stage: staging script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy environment: name: staging url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN @@ -81,15 +84,15 @@ staging: # CANARY_ENABLED. canary: + extends: .auto-deploy stage: canary script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy canary + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy canary environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -102,18 +105,18 @@ canary: - $CANARY_ENABLED .production: &production_template + extends: .auto-deploy stage: production script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy - - delete canary - - delete rollout - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy delete canary + - auto-deploy delete rollout + - auto-deploy persist_environment_url environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -152,17 +155,17 @@ production_manual: # This job implements incremental rollout on for every push to `master`. .rollout: &rollout_template + extends: .auto-deploy script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy rollout $ROLLOUT_PERCENTAGE - - scale stable $((100-ROLLOUT_PERCENTAGE)) - - delete canary - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE + - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) + - auto-deploy delete canary + - auto-deploy persist_environment_url environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -240,330 +243,3 @@ rollout 100%: <<: *manual_rollout_template <<: *production_template allow_failure: false - -.deploy_helpers: &deploy_helpers | - [[ "$TRACE" ]] && set -x - auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} - export DATABASE_URL=${DATABASE_URL-$auto_database_url} - export TILLER_NAMESPACE=$KUBE_NAMESPACE - - function get_replicas() { - track="${1:-stable}" - percentage="${2:-100}" - - env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' ) - env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' ) - - if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then - # for stable track get number of replicas from `PRODUCTION_REPLICAS` - eval new_replicas=\$${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - new_replicas=$REPLICAS - fi - else - # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` - eval new_replicas=\$${env_track}_${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - eval new_replicas=\${env_track}_REPLICAS - fi - fi - - replicas="${new_replicas:-1}" - replicas="$(($replicas * $percentage / 100))" - - # always return at least one replicas - if [[ $replicas -gt 0 ]]; then - echo "$replicas" - else - echo 1 - fi - } - - # Extracts variables prefixed with K8S_SECRET_ - # and creates a Kubernetes secret. - # - # e.g. If we have the following environment variables: - # K8S_SECRET_A=value1 - # K8S_SECRET_B=multi\ word\ value - # - # Then we will create a secret with the following key-value pairs: - # data: - # A: dmFsdWUxCg== - # B: bXVsdGkgd29yZCB2YWx1ZQo= - function create_application_secret() { - track="${1-stable}" - export APPLICATION_SECRET_NAME=$(application_secret_name "$track") - - env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables - - kubectl create secret \ - -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \ - --from-env-file k8s_prefixed_variables -o yaml --dry-run | - kubectl replace -n "$KUBE_NAMESPACE" --force -f - - - export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1) - - rm k8s_prefixed_variables - } - - function deploy_name() { - name="$CI_ENVIRONMENT_SLUG" - track="${1-stable}" - - if [[ "$track" != "stable" ]]; then - name="$name-$track" - fi - - echo $name - } - - function application_secret_name() { - track="${1-stable}" - name=$(deploy_name "$track") - - echo "${name}-secret" - } - - function deploy() { - track="${1-stable}" - percentage="${2:-100}" - name=$(deploy_name "$track") - - if [[ -z "$CI_COMMIT_TAG" ]]; then - image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG} - image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA} - else - image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE} - image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} - fi - - service_enabled="true" - postgres_enabled="$POSTGRES_ENABLED" - - # if track is different than stable, - # re-use all attached resources - if [[ "$track" != "stable" ]]; then - service_enabled="false" - postgres_enabled="false" - fi - - replicas=$(get_replicas "$track" "$percentage") - - if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then - secret_name='gitlab-registry' - else - secret_name='' - fi - - create_application_secret "$track" - - env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]') - eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS - if [ -n "$env_ADDITIONAL_HOSTS" ]; then - additional_hosts="{$env_ADDITIONAL_HOSTS}" - elif [ -n "$ADDITIONAL_HOSTS" ]; then - additional_hosts="{$ADDITIONAL_HOSTS}" - fi - - if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then - echo "Deploying first release with database initialization..." - helm upgrade --install \ - --wait \ - --set service.enabled="$service_enabled" \ - --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ - --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ - --set image.repository="$image_repository" \ - --set image.tag="$image_tag" \ - --set image.pullPolicy=IfNotPresent \ - --set image.secrets[0].name="$secret_name" \ - --set application.track="$track" \ - --set application.database_url="$DATABASE_URL" \ - --set application.secretName="$APPLICATION_SECRET_NAME" \ - --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \ - --set service.url="$CI_ENVIRONMENT_URL" \ - --set service.additionalHosts="$additional_hosts" \ - --set replicaCount="$replicas" \ - --set postgresql.enabled="$postgres_enabled" \ - --set postgresql.nameOverride="postgres" \ - --set postgresql.postgresUser="$POSTGRES_USER" \ - --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ - --set postgresql.postgresDatabase="$POSTGRES_DB" \ - --set postgresql.imageTag="$POSTGRES_VERSION" \ - --set application.initializeCommand="$DB_INITIALIZE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - - echo "Deploying second release..." - helm upgrade --reuse-values \ - --wait \ - --set application.initializeCommand="" \ - --set application.migrateCommand="$DB_MIGRATE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - else - echo "Deploying new release..." - helm upgrade --install \ - --wait \ - --set service.enabled="$service_enabled" \ - --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ - --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ - --set image.repository="$image_repository" \ - --set image.tag="$image_tag" \ - --set image.pullPolicy=IfNotPresent \ - --set image.secrets[0].name="$secret_name" \ - --set application.track="$track" \ - --set application.database_url="$DATABASE_URL" \ - --set application.secretName="$APPLICATION_SECRET_NAME" \ - --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \ - --set service.url="$CI_ENVIRONMENT_URL" \ - --set service.additionalHosts="$additional_hosts" \ - --set replicaCount="$replicas" \ - --set postgresql.enabled="$postgres_enabled" \ - --set postgresql.nameOverride="postgres" \ - --set postgresql.postgresUser="$POSTGRES_USER" \ - --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ - --set postgresql.postgresDatabase="$POSTGRES_DB" \ - --set postgresql.imageTag="$POSTGRES_VERSION" \ - --set application.migrateCommand="$DB_MIGRATE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - fi - - if [[ -z "$ROLLOUT_STATUS_DISABLED" ]]; then - kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name" - fi - } - - function scale() { - track="${1-stable}" - percentage="${2-100}" - name=$(deploy_name "$track") - - replicas=$(get_replicas "$track" "$percentage") - - if [[ -n "$(helm ls -q "^$name$")" ]]; then - helm upgrade --reuse-values \ - --wait \ - --set replicaCount="$replicas" \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - fi - } - - function install_dependencies() { - apk add -U openssl curl tar gzip bash ca-certificates git - curl -sSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub - curl -sSL -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk - apk add glibc-2.28-r0.apk - rm glibc-2.28-r0.apk - - curl -sS "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx - mv linux-amd64/helm /usr/bin/ - mv linux-amd64/tiller /usr/bin/ - helm version --client - tiller -version - - curl -sSL -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" - chmod +x /usr/bin/kubectl - kubectl version --client - } - - function download_chart() { - if [[ ! -d chart ]]; then - auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app} - auto_chart_name=$(basename $auto_chart) - auto_chart_name=${auto_chart_name%.tgz} - auto_chart_name=${auto_chart_name%.tar.gz} - else - auto_chart="chart" - auto_chart_name="chart" - fi - - helm init --client-only - helm repo add ${AUTO_DEVOPS_CHART_REPOSITORY_NAME:-gitlab} ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} ${AUTO_DEVOPS_CHART_REPOSITORY_USERNAME:+"--username" "$AUTO_DEVOPS_CHART_REPOSITORY_USERNAME"} ${AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD:+"--password" "$AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD"} - if [[ ! -d "$auto_chart" ]]; then - helm fetch ${auto_chart} --untar - fi - if [ "$auto_chart_name" != "chart" ]; then - mv ${auto_chart_name} chart - fi - - helm dependency update chart/ - helm dependency build chart/ - } - - function ensure_namespace() { - kubectl get namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" - } - - function check_kube_domain() { - if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then - echo "In order to deploy or use Review Apps," - echo "KUBE_INGRESS_BASE_DOMAIN variables must be set" - echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings" - echo "or by defining a variable at group or project level." - echo "You can also manually add it in .gitlab-ci.yml" - false - else - true - fi - } - - function initialize_tiller() { - echo "Checking Tiller..." - - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 & - echo "Tiller is listening on ${HELM_HOST}" - - if ! helm version --debug; then - echo "Failed to init Tiller." - return 1 - fi - echo "" - } - - function create_secret() { - echo "Create secret..." - if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then - return - fi - - kubectl create secret -n "$KUBE_NAMESPACE" \ - docker-registry gitlab-registry \ - --docker-server="$CI_REGISTRY" \ - --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \ - --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \ - --docker-email="$GITLAB_USER_EMAIL" \ - -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f - - } - - function persist_environment_url() { - echo $CI_ENVIRONMENT_URL > environment_url.txt - } - - function delete() { - track="${1-stable}" - name=$(deploy_name "$track") - - if [[ -n "$(helm ls -q "^$name$")" ]]; then - helm delete --purge "$name" - fi - - secret_name=$(application_secret_name "$track") - kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name" - } - -before_script: - - *deploy_helpers diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index 13ab98d3a16..84bb0ff3b33 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -1,5 +1,3 @@ -# This file is a template, and might need editing before it works on your project. - # Build JAVA applications using Apache Maven (http://maven.apache.org) # For docker image tags see https://hub.docker.com/_/maven/ # diff --git a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml index 83e179f37c3..0a3cf3dcf77 100644 --- a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml @@ -1,5 +1,5 @@ image: - name: hashicorp/packer:1.0.4 + name: hashicorp/packer:latest entrypoint: - '/usr/bin/env' - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 5ad624bb15f..2afc99d0bf8 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -5,6 +5,7 @@ container_scanning: image: docker:stable variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" # Defining two new variables based on GitLab's CI/CD predefined variables # https://docs.gitlab.com/ee/ci/variables/#predefined-environment-variables CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG @@ -22,8 +23,9 @@ container_scanning: DOCKER_SERVICE: docker DOCKER_HOST: tcp://${DOCKER_SERVICE}:2375/ # https://hub.docker.com/r/arminc/clair-local-scan/tags - CLAIR_LOCAL_SCAN_VERSION: v2.0.8_fe9b059d930314b54c78f75afe265955faf4fdc1 - CLAIR_EXECUTABLE_VERSION: v11 + CLAIR_LOCAL_SCAN_VERSION: v2.0.8_0ed98e9ead65a51ba53f7cc53fa5e80c92169207 + CLAIR_EXECUTABLE_VERSION: v12 + CLAIR_EXECUTABLE_SHA: 44f2a3fdd7b0d102c98510e7586f6956edc89ab72c6943980f92f4979f7f4081 ## Disable the proxy for clair-local-scan, otherwise Container Scanning will ## fail when a proxy is used. NO_PROXY: ${DOCKER_SERVICE},localhost @@ -43,6 +45,7 @@ container_scanning: - apk add -U wget ca-certificates - docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} - wget https://github.com/arminc/clair-scanner/releases/download/${CLAIR_EXECUTABLE_VERSION}/clair-scanner_linux_amd64 + - echo "${CLAIR_EXECUTABLE_SHA} clair-scanner_linux_amd64" | sha256sum -c - mv clair-scanner_linux_amd64 clair-scanner - chmod +x clair-scanner - touch clair-whitelist.yml diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index f176771775e..15b84f1540d 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -9,6 +9,7 @@ dependency_scanning: image: docker:stable variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" allow_failure: true services: - docker:stable-dind @@ -41,6 +42,9 @@ dependency_scanning: DS_PULL_ANALYZER_IMAGE_TIMEOUT \ DS_RUN_ANALYZER_TIMEOUT \ DS_PYTHON_VERSION \ + DS_PIP_DEPENDENCY_PATH \ + PIP_INDEX_URL \ + PIP_EXTRA_INDEX_URL \ ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 0a97a16b83c..90278122361 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -9,6 +9,7 @@ sast: image: docker:stable variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" allow_failure: true services: - docker:stable-dind @@ -45,11 +46,14 @@ sast: SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ SAST_RUN_ANALYZER_TIMEOUT \ + SAST_JAVA_VERSION \ ANT_HOME \ ANT_PATH \ GRADLE_PATH \ JAVA_OPTS \ JAVA_PATH \ + JAVA_8_VERSION \ + JAVA_11_VERSION \ MAVEN_CLI_OPTS \ MAVEN_PATH \ MAVEN_REPO_PATH \ diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml index a3db2705bf6..280e75d46f5 100644 --- a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml @@ -8,26 +8,23 @@ stages: - deploy .serverless:build:image: - stage: build image: registry.gitlab.com/gitlab-org/gitlabktl:latest + stage: build script: /usr/bin/gitlabktl app build .serverless:deploy:image: + image: registry.gitlab.com/gitlab-org/gitlabktl:latest stage: deploy - image: gcr.io/triggermesh/tm@sha256:3cfdd470a66b741004fb02354319d79f1598c70117ce79978d2e07e192bfb336 # v0.0.11 environment: development - script: - - echo "$CI_REGISTRY_IMAGE" - - tm -n "$KUBE_NAMESPACE" --config "$KUBECONFIG" deploy service "$CI_PROJECT_NAME" --from-image "$CI_REGISTRY_IMAGE" --wait + script: /usr/bin/gitlabktl app deploy .serverless:build:functions: - stage: build - environment: development image: registry.gitlab.com/gitlab-org/gitlabktl:latest + stage: build script: /usr/bin/gitlabktl serverless build .serverless:deploy:functions: + image: registry.gitlab.com/gitlab-org/gitlabktl:latest stage: deploy environment: development - image: registry.gitlab.com/gitlab-org/gitlabktl:latest script: /usr/bin/gitlabktl serverless deploy diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index ce5857965bf..9550bc6d39c 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -63,7 +63,15 @@ module Gitlab end def exist? - trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present? + archived_trace_exist? || live_trace_exist? + end + + def archived_trace_exist? + trace_artifact&.exists? + end + + def live_trace_exist? + job.trace_chunks.any? || current_path.present? || old_trace.present? end def read @@ -118,7 +126,7 @@ module Gitlab raise AlreadyArchivedError, 'Could not write to the archived trace' elsif current_path File.open(current_path, mode) - elsif Feature.enabled?('ci_enable_live_trace') + elsif Feature.enabled?('ci_enable_live_trace', job.project) Gitlab::Ci::Trace::ChunkedIO.new(job) else File.open(ensure_path, mode) @@ -167,7 +175,7 @@ module Gitlab def clone_file!(src_stream, temp_dir) FileUtils.mkdir_p(temp_dir) - Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path| + Dir.mktmpdir("tmp-trace-#{job.id}", temp_dir) do |dir_path| temp_path = File.join(dir_path, "job.log") FileUtils.touch(temp_path) size = IO.copy_stream(src_stream, temp_path) diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 8c6fd56493f..e99889f4a25 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -166,6 +166,13 @@ module Gitlab end def destroy! + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless build.has_archived_trace? + Sidekiq.logger.warn(message: 'The job does not have archived trace but going to be destroyed.', + job_id: build.id) + end + trace_chunks.fast_destroy_all @tell = @size = 0 ensure diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index a5693dc4f81..2e1eab270ff 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -40,6 +40,7 @@ module Gitlab environment: job[:environment_name], coverage_regex: job[:coverage], yaml_variables: yaml_variables(name), + needs_attributes: job[:needs]&.map { |need| { name: need } }, options: { image: job[:image], services: job[:services], @@ -54,7 +55,8 @@ module Gitlab parallel: job[:parallel], instance: job[:instance], start_in: job[:start_in], - trigger: job[:trigger] + trigger: job[:trigger], + bridge_needs: job[:needs] }.compact }.compact end @@ -108,6 +110,7 @@ module Gitlab validate_job_stage!(name, job) validate_job_dependencies!(name, job) + validate_job_needs!(name, job) validate_job_environment!(name, job) end end @@ -144,12 +147,30 @@ module Gitlab job[:dependencies].each do |dependency| raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] - unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index + dependency_stage_index = @stages.index(@jobs[dependency.to_sym][:stage]) + + unless dependency_stage_index.present? && dependency_stage_index < stage_index raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" end end end + def validate_job_needs!(name, job) + return unless job[:needs] + + stage_index = @stages.index(job[:stage]) + + job[:needs].each do |need| + raise ValidationError, "#{name} job: undefined need: #{need}" unless @jobs[need.to_sym] + + needs_stage_index = @stages.index(@jobs[need.to_sym][:stage]) + + unless needs_stage_index.present? && needs_stage_index < stage_index + raise ValidationError, "#{name} job: need #{need} is not defined in prior stages" + end + end + end + def validate_job_environment!(name, job) return unless job[:environment] return unless job[:environment].is_a?(Hash) diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files.rb b/lib/gitlab/cleanup/orphan_job_artifact_files.rb index ee7164b3e55..808814c39e0 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files.rb @@ -17,7 +17,7 @@ module Gitlab @limit = limit @dry_run = dry_run @niceness = niceness || DEFAULT_NICENESS - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger @total_found = @total_cleaned = 0 new_batch! diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb b/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb index 5c30258c0fc..53e0c83046e 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb @@ -22,7 +22,7 @@ module Gitlab attr_reader :batch_size, :dry_run attr_accessor :artifact_files - def initialize(batch_size:, dry_run: true, logger: Rails.logger) + def initialize(batch_size:, dry_run: true, logger: Rails.logger) # rubocop:disable Gitlab/RailsLogger @batch_size = batch_size @dry_run = dry_run @logger = logger diff --git a/lib/gitlab/cleanup/project_upload_file_finder.rb b/lib/gitlab/cleanup/project_upload_file_finder.rb index 2ee8b60e76a..5aace564c2d 100644 --- a/lib/gitlab/cleanup/project_upload_file_finder.rb +++ b/lib/gitlab/cleanup/project_upload_file_finder.rb @@ -49,7 +49,7 @@ module Gitlab cmd = %W[#{ionice} -c Idle] + cmd if ionice log_msg = "find command: \"#{cmd.join(' ')}\"" - Rails.logger.info log_msg + Rails.logger.info log_msg # rubocop:disable Gitlab/RailsLogger cmd end diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb index 82a405362c2..056e075cb21 100644 --- a/lib/gitlab/cleanup/project_uploads.rb +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -8,7 +8,7 @@ module Gitlab attr_reader :logger def initialize(logger: nil) - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger end def run!(dry_run: true) diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb index 03298d960a4..42c93b7aecb 100644 --- a/lib/gitlab/cleanup/remote_uploads.rb +++ b/lib/gitlab/cleanup/remote_uploads.rb @@ -7,7 +7,7 @@ module Gitlab BATCH_SIZE = 100 def initialize(logger: nil) - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger end def run!(dry_run: false) diff --git a/lib/gitlab/cluster/puma_worker_killer_observer.rb b/lib/gitlab/cluster/puma_worker_killer_observer.rb index 3b4ebc3fbae..f53051c32ff 100644 --- a/lib/gitlab/cluster/puma_worker_killer_observer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_observer.rb @@ -15,9 +15,7 @@ module Gitlab private def log_termination(worker) - labels = { worker: "worker_#{worker.index}" } - - @counter.increment(labels) + @counter.increment end end end diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb index 560fe63df0e..87bd257f69a 100644 --- a/lib/gitlab/config/entry/attributable.rb +++ b/lib/gitlab/config/entry/attributable.rb @@ -18,6 +18,10 @@ module Gitlab config[attribute] end + + define_method("has_#{attribute}?") do + config.is_a?(Hash) && config.key?(attribute) + end end end end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 6796fcce75f..374f929878e 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -20,7 +20,20 @@ module Gitlab present_keys = value.try(:keys).to_a & options[:in] if present_keys.any? - record.errors.add(attribute, "contains disallowed keys: " + + message = options[:message] || "contains disallowed keys" + message += ": #{present_keys.join(', ')}" + + record.errors.add(attribute, message) + end + end + end + + class RequiredKeysValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + present_keys = options[:in] - value.try(:keys).to_a + + if present_keys.any? + record.errors.add(attribute, "missing required keys: " + present_keys.join(', ')) end end @@ -54,6 +67,16 @@ module Gitlab end end + class ArrayOfHashesValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless value.is_a?(Array) && value.map { |hsh| hsh.is_a?(Hash) }.all? + record.errors.add(attribute, 'should be an array of hashes') + end + end + end + class ArrayOrStringValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) unless value.is_a?(Array) || value.is_a?(String) @@ -220,6 +243,14 @@ module Gitlab end end + class ExpressionValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(String) && ::Gitlab::Ci::Pipeline::Expression::Statement.new(value).valid? + record.errors.add(attribute, 'Invalid expression syntax') + end + end + end + class PortNamePresentAndUniqueValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) return unless value.is_a?(Array) diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb new file mode 100644 index 00000000000..ff844645b11 --- /dev/null +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module ContentSecurityPolicy + class ConfigLoader + DIRECTIVES = %w(base_uri child_src connect_src default_src font_src + form_action frame_ancestors frame_src img_src manifest_src + media_src object_src report_uri script_src style_src worker_src).freeze + + def self.default_settings_hash + { + 'enabled' => false, + 'report_only' => false, + 'directives' => DIRECTIVES.each_with_object({}) { |directive, hash| hash[directive] = nil } + } + end + + def initialize(csp_directives) + @csp_directives = HashWithIndifferentAccess.new(csp_directives) + end + + def load(policy) + DIRECTIVES.each do |directive| + arguments = arguments_for(directive) + + next unless arguments.present? + + policy.public_send(directive, *arguments) # rubocop:disable GitlabSecurity/PublicSend + end + end + + private + + def arguments_for(directive) + arguments = @csp_directives[directive.to_s] + + return unless arguments.present? && arguments.is_a?(String) + + arguments.strip.split(' ').map(&:strip) + end + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index f7d046600e8..5b0b91de5da 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -84,11 +84,7 @@ module Gitlab .and(t[:created_at].lteq(Date.current.end_of_day)) .and(t[:author_id].eq(contributor.id)) - date_interval = if Gitlab::Database.postgresql? - "INTERVAL '#{Time.zone.now.utc_offset} seconds'" - else - "INTERVAL #{Time.zone.now.utc_offset} SECOND" - end + date_interval = "INTERVAL '#{Time.zone.now.utc_offset} seconds'" Event.reorder(nil) .select(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval}) AS date", 'count(id) as total_amount') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 469a7fd9f7b..6ce47650562 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -7,6 +7,11 @@ module Gitlab Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } end + def expire_current_application_settings + ::ApplicationSetting.expire + Gitlab::SafeRequestStore.delete(:current_application_settings) + end + def clear_in_memory_application_settings! @in_memory_application_settings = nil end @@ -45,7 +50,7 @@ module Gitlab # need to be added to the application settings. To prevent Rake tasks # and other callers from failing, use any loaded settings and return # defaults for missing columns. - if ActiveRecord::Migrator.needs_migration? + if ActiveRecord::Base.connection.migration_context.needs_migration? db_attributes = current_settings&.attributes || {} fake_application_settings(db_attributes) elsif current_settings.present? diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index 304d60996a6..07ae430c45e 100644 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -4,13 +4,13 @@ module Gitlab module CycleAnalytics class BaseEventFetcher include BaseQuery + include GroupProjectsProvider - attr_reader :projections, :query, :stage, :order + attr_reader :projections, :query, :stage, :order, :options MAX_EVENTS = 50 - def initialize(project:, stage:, options:) - @project = project + def initialize(stage:, options:) @stage = stage @options = options end @@ -40,13 +40,13 @@ module Gitlab end def events_query - diff_fn = subtract_datetimes_diff(base_query, @options[:start_time_attrs], @options[:end_time_attrs]) + diff_fn = subtract_datetimes_diff(base_query, options[:start_time_attrs], options[:end_time_attrs]) base_query.project(extract_diff_epoch(diff_fn).as('total_time'), *projections).order(order.desc).take(MAX_EVENTS) end def default_order - [@options[:start_time_attrs]].flatten.first + [options[:start_time_attrs]].flatten.first end def serialize(_event) @@ -59,13 +59,21 @@ module Gitlab def allowed_ids @allowed_ids ||= allowed_ids_finder_class - .new(@options[:current_user], project_id: @project.id) + .new(options[:current_user], allowed_ids_source) .execute.where(id: event_result_ids).pluck(:id) end def event_result_ids event_result.map { |event| event['id'] } end + + def allowed_ids_source + group ? { group_id: group.id, include_subgroups: true } : { project_id: project.id } + end + + def serialization_context + {} + end end end end diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index 36231b187cd..459bb5177b5 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -10,23 +10,38 @@ module Gitlab private def base_query - @base_query ||= stage_query(@project.id) # rubocop:disable Gitlab/ModuleWithInstanceVariables + @base_query ||= stage_query(projects.map(&:id)) end def stage_query(project_ids) query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])) .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) + .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) + .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) .project(issue_table[:project_id].as("project_id")) - .where(issue_table[:project_id].in(project_ids)) - .where(issue_table[:created_at].gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables + .project(projects_table[:path].as("project_path")) + .project(routes_table[:path].as("namespace_path")) + + query = limit_query(query, project_ids) # Load merge_requests - query = query.join(mr_table, Arel::Nodes::OuterJoin) + + query = load_merge_requests(query) + + query + end + + def limit_query(query, project_ids) + query.where(issue_table[:project_id].in(project_ids)) + .where(routes_table[:source_type].eq('Namespace')) + .where(issue_table[:created_at].gteq(options[:from])) + end + + def load_merge_requests(query) + query.join(mr_table, Arel::Nodes::OuterJoin) .on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])) .join(mr_metrics_table) .on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) - - query end end end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index e2d6a301734..1cd54238bb4 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -4,9 +4,11 @@ module Gitlab module CycleAnalytics class BaseStage include BaseQuery + include GroupProjectsProvider - def initialize(project:, options:) - @project = project + attr_reader :options + + def initialize(options:) @options = options end @@ -14,30 +16,23 @@ module Gitlab event_fetcher.fetch end - def as_json - AnalyticsStageSerializer.new.represent(self) + def as_json(serializer: AnalyticsStageSerializer) + serializer.new.represent(self) end def title raise NotImplementedError.new("Expected #{self.name} to implement title") end - def median - BatchLoader.for(@project.id).batch(key: name) do |project_ids, loader| - cte_table = Arel::Table.new("cte_table_for_#{name}") - - # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). - # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). - # We compute the (end_time - start_time) interval, and give it an alias based on the current - # cycle analytics stage. - interval_query = Arel::Nodes::As.new(cte_table, - subtract_datetimes(stage_query(project_ids), start_time_attrs, end_time_attrs, name.to_s)) + def project_median + return if project.nil? + BatchLoader.for(project.id).batch(key: name) do |project_ids, loader| if project_ids.one? - loader.call(@project.id, median_datetime(cte_table, interval_query, name)) + loader.call(project.id, median_query(project_ids)) else begin - median_datetimes(cte_table, interval_query, name, :project_id)&.each do |project_id, median| + median_datetimes(cte_table, interval_query(project_ids), name, :project_id)&.each do |project_id, median| loader.call(project_id, median) end rescue NotSupportedError @@ -47,20 +42,41 @@ module Gitlab end end + def group_median + median_query(projects.map(&:id)) + end + + def median_query(project_ids) + # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). + # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). + # We compute the (end_time - start_time) interval, and give it an alias based on the current + # cycle analytics stage. + + median_datetime(cte_table, interval_query(project_ids), name) + end + def name raise NotImplementedError.new("Expected #{self.name} to implement name") end + def cte_table + Arel::Table.new("cte_table_for_#{name}") + end + + def interval_query(project_ids) + Arel::Nodes::As.new(cte_table, + subtract_datetimes(stage_query(project_ids), start_time_attrs, end_time_attrs, name.to_s)) + end + private def event_fetcher - @event_fetcher ||= Gitlab::CycleAnalytics::EventFetcher[name].new(project: @project, - stage: name, + @event_fetcher ||= Gitlab::CycleAnalytics::EventFetcher[name].new(stage: name, options: event_options) end def event_options - @options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) + options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) end end end diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb index 6c348f1862d..fcc282bf7a6 100644 --- a/lib/gitlab/cycle_analytics/code_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb @@ -20,7 +20,7 @@ module Gitlab private def serialize(event) - AnalyticsMergeRequestSerializer.new(project: @project).represent(event) + AnalyticsMergeRequestSerializer.new(serialization_context).represent(event) end def allowed_ids_finder_class diff --git a/lib/gitlab/cycle_analytics/group_projects_provider.rb b/lib/gitlab/cycle_analytics/group_projects_provider.rb new file mode 100644 index 00000000000..1287a48daaa --- /dev/null +++ b/lib/gitlab/cycle_analytics/group_projects_provider.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module GroupProjectsProvider + def projects + group ? projects_for_group : [project] + end + + def group + @group ||= options.fetch(:group, nil) + end + + def project + @project ||= options.fetch(:project, nil) + end + + private + + def projects_for_group + projects = Project.inside_path(group.full_path) + projects = projects.where(id: options[:projects]) if options[:projects] + projects + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/group_stage_summary.rb b/lib/gitlab/cycle_analytics/group_stage_summary.rb new file mode 100644 index 00000000000..a1fc941495d --- /dev/null +++ b/lib/gitlab/cycle_analytics/group_stage_summary.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + class GroupStageSummary + attr_reader :group, :from, :current_user, :options + + def initialize(group, options:) + @group = group + @from = options[:from] + @current_user = options[:current_user] + @options = options + end + + def data + [serialize(Summary::Group::Issue.new(group: group, from: from, current_user: current_user, options: options)), + serialize(Summary::Group::Deploy.new(group: group, from: from, options: options))] + end + + private + + def serialize(summary_object) + AnalyticsSummarySerializer.new.represent(summary_object) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb index 8a870f2e2a3..6914cf24c19 100644 --- a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb @@ -18,7 +18,7 @@ module Gitlab private def serialize(event) - AnalyticsIssueSerializer.new(project: @project).represent(event) + AnalyticsIssueSerializer.new(serialization_context).represent(event) end def allowed_ids_finder_class diff --git a/lib/gitlab/cycle_analytics/issue_helper.rb b/lib/gitlab/cycle_analytics/issue_helper.rb index c9266341378..295eca5edca 100644 --- a/lib/gitlab/cycle_analytics/issue_helper.rb +++ b/lib/gitlab/cycle_analytics/issue_helper.rb @@ -5,13 +5,23 @@ module Gitlab module IssueHelper def stage_query(project_ids) query = issue_table.join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) + .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) + .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) .project(issue_table[:project_id].as("project_id")) - .where(issue_table[:project_id].in(project_ids)) - .where(issue_table[:created_at].gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables - .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + .project(projects_table[:path].as("project_path")) + .project(routes_table[:path].as("namespace_path")) + + query = limit_query(query, project_ids) query end + + def limit_query(query, project_ids) + query.where(issue_table[:project_id].in(project_ids)) + .where(routes_table[:source_type].eq('Namespace')) + .where(issue_table[:created_at].gteq(options[:from])) + .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + end end end end diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb index 3e0302d308d..015f7bfde24 100644 --- a/lib/gitlab/cycle_analytics/metrics_tables.rb +++ b/lib/gitlab/cycle_analytics/metrics_tables.rb @@ -35,6 +35,14 @@ module Gitlab User.arel_table end + def projects_table + Project.arel_table + end + + def routes_table + Route.arel_table + end + def build_table ::CommitStatus.arel_table end diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb index afefd09b614..55214e6b896 100644 --- a/lib/gitlab/cycle_analytics/permissions.rb +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -23,7 +23,7 @@ module Gitlab end def get - ::CycleAnalytics::STAGES.each do |stage| + ::CycleAnalytics::LevelBase::STAGES.each do |stage| @stage_permission_hash[stage] = authorized_stage?(stage) end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index d924f956dcd..bad02e00a13 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -18,7 +18,7 @@ module Gitlab private def serialize(event) - AnalyticsIssueSerializer.new(project: @project).represent(event) + AnalyticsIssueSerializer.new(serialization_context).represent(event) end def allowed_ids_finder_class diff --git a/lib/gitlab/cycle_analytics/plan_helper.rb b/lib/gitlab/cycle_analytics/plan_helper.rb index 30fc2ce6d40..a63ae58ad21 100644 --- a/lib/gitlab/cycle_analytics/plan_helper.rb +++ b/lib/gitlab/cycle_analytics/plan_helper.rb @@ -5,14 +5,23 @@ module Gitlab module PlanHelper def stage_query(project_ids) query = issue_table.join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) + .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) + .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) .project(issue_table[:project_id].as("project_id")) + .project(projects_table[:path].as("project_path")) + .project(routes_table[:path].as("namespace_path")) .where(issue_table[:project_id].in(project_ids)) - .where(issue_table[:created_at].gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables - .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) - .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil)) + .where(routes_table[:source_type].eq('Namespace')) + query = limit_query(query) query end + + def limit_query(query) + query.where(issue_table[:created_at].gteq(options[:from])) + .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil)) + end end end end diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb index 6bcbe0412a9..8843ab2bcb9 100644 --- a/lib/gitlab/cycle_analytics/production_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/production_event_fetcher.rb @@ -10,7 +10,8 @@ module Gitlab issue_table[:iid], issue_table[:id], issue_table[:created_at], - issue_table[:author_id]] + issue_table[:author_id], + routes_table[:path]] super(*args) end @@ -18,7 +19,7 @@ module Gitlab private def serialize(event) - AnalyticsIssueSerializer.new(project: @project).represent(event) + AnalyticsIssueSerializer.new(serialization_context).represent(event) end def allowed_ids_finder_class diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb index aff65b150fb..778757a9ede 100644 --- a/lib/gitlab/cycle_analytics/production_helper.rb +++ b/lib/gitlab/cycle_analytics/production_helper.rb @@ -6,7 +6,7 @@ module Gitlab def stage_query(project_ids) super(project_ids) .where(mr_metrics_table[:first_deployed_to_production_at] - .gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables + .gteq(options[:from])) end end end diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb index b6354b5ffad..4b5d79097b7 100644 --- a/lib/gitlab/cycle_analytics/review_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb @@ -19,7 +19,7 @@ module Gitlab private def serialize(event) - AnalyticsMergeRequestSerializer.new(project: @project).represent(event) + AnalyticsMergeRequestSerializer.new(serialization_context).represent(event) end def allowed_ids_finder_class diff --git a/lib/gitlab/cycle_analytics/summary/group/base.rb b/lib/gitlab/cycle_analytics/summary/group/base.rb new file mode 100644 index 00000000000..48d8164bde1 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/group/base.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + module Group + class Base + attr_reader :group, :from, :options + + def initialize(group:, from:, options:) + @group = group + @from = from + @options = options + end + + def title + raise NotImplementedError.new("Expected #{self.name} to implement title") + end + + def value + raise NotImplementedError.new("Expected #{self.name} to implement value") + end + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/group/deploy.rb b/lib/gitlab/cycle_analytics/summary/group/deploy.rb new file mode 100644 index 00000000000..78d677cf558 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/group/deploy.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + module Group + class Deploy < Group::Base + include GroupProjectsProvider + + def title + n_('Deploy', 'Deploys', value) + end + + def value + @value ||= find_deployments + end + + private + + def find_deployments + deployments = Deployment.joins(:project).merge(Project.inside_path(group.full_path)) + deployments = deployments.where(projects: { id: options[:projects] }) if options[:projects] + deployments = deployments.where("deployments.created_at > ?", from) + deployments.success.count + end + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/group/issue.rb b/lib/gitlab/cycle_analytics/summary/group/issue.rb new file mode 100644 index 00000000000..9daae8531d8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/group/issue.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + module Group + class Issue < Group::Base + attr_reader :group, :from, :current_user, :options + + def initialize(group:, from:, current_user:, options:) + @group = group + @from = from + @current_user = current_user + @options = options + end + + def title + n_('New Issue', 'New Issues', value) + end + + def value + @value ||= find_issues + end + + private + + def find_issues + issues = IssuesFinder.new(current_user, group_id: group.id, include_subgroups: true, created_after: from).execute + issues = issues.where(projects: { id: options[:projects] }) if options[:projects] + issues.count + end + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/test_helper.rb b/lib/gitlab/cycle_analytics/test_helper.rb index 32fca7fa898..d9124d62c7c 100644 --- a/lib/gitlab/cycle_analytics/test_helper.rb +++ b/lib/gitlab/cycle_analytics/test_helper.rb @@ -14,7 +14,7 @@ module Gitlab private def branch - @branch ||= @options[:branch] # rubocop:disable Gitlab/ModuleWithInstanceVariables + @branch ||= options[:branch] end end end diff --git a/lib/gitlab/cycle_analytics/usage_data.rb b/lib/gitlab/cycle_analytics/usage_data.rb index 913ee373f54..644300caead 100644 --- a/lib/gitlab/cycle_analytics/usage_data.rb +++ b/lib/gitlab/cycle_analytics/usage_data.rb @@ -32,7 +32,7 @@ module Gitlab def medians_per_stage projects.each_with_object({}) do |project, hsh| - ::CycleAnalytics.new(project, options).all_medians_per_stage.each do |stage_name, median| + ::CycleAnalytics::ProjectLevel.new(project, options: options).all_medians_by_stage.each do |stage_name, median| hsh[stage_name] ||= [] hsh[stage_name] << median end diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index 6d5fc4219fb..2f4ae010e74 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -46,7 +46,10 @@ module Gitlab if thread thread.wakeup if thread.alive? - thread.join unless Thread.current == thread + begin + thread.join unless Thread.current == thread + rescue Exception # rubocop:disable Lint/RescueException + end @thread = nil end end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 0fc145534bf..5424298723e 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -46,6 +46,16 @@ module Gitlab ee? ? 'gitlab-ee' : 'gitlab-ce' end + def markdown_list(items) + list = items.map { |item| "* `#{item}`" }.join("\n") + + if items.size > 10 + "\n<details>\n\n#{list}\n\n</details>\n" + else + list + end + end + # @return [Hash<String,Array<String>>] def changes_by_category all_changed_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, hash| @@ -103,7 +113,7 @@ module Gitlab yarn\.lock )\z}x => :frontend, - %r{\A(ee/)?db/} => :database, + %r{\A(ee/)?db/(?!fixtures)[^/]+} => :database, %r{\A(ee/)?lib/gitlab/(database|background_migration|sql|github_import)(/|\.rb)} => :database, %r{\A(app/models/project_authorization|app/services/users/refresh_authorized_projects_service)(/|\.rb)} => :database, %r{\Arubocop/cop/migration(/|\.rb)} => :database, @@ -116,6 +126,7 @@ module Gitlab %r{\A(ee/)?vendor/(languages\.yml|licenses\.csv)\z} => :backend, %r{\A(Dangerfile|Gemfile|Gemfile.lock|Procfile|Rakefile|\.gitlab-ci\.yml)\z} => :backend, %r{\A[A-Z_]+_VERSION\z} => :backend, + %r{\A\.rubocop(_todo)?\.yml\z} => :backend, %r{\A(ee/)?qa/} => :qa, @@ -132,6 +143,22 @@ module Gitlab def new_teammates(usernames) usernames.map { |u| Gitlab::Danger::Teammate.new('username' => u) } end + + def missing_database_labels(current_mr_labels) + labels = if has_database_scoped_labels?(current_mr_labels) + ['database'] + else + ['database', 'database::review pending'] + end + + labels - current_mr_labels + end + + private + + def has_database_scoped_labels?(current_mr_labels) + current_mr_labels.any? { |label| label.start_with?('database::') } + end end end end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index b44f134f2c1..8ae4ffdadb0 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -39,9 +39,9 @@ module Gitlab def has_capability?(project, category, kind, labels) case category when :test - area = role[/Test Automation Engineer, (\w+)/, 1] + area = role[/Test Automation Engineer(?:.*?, (\w+))/, 1].downcase - area && labels.any?(area) if kind == :reviewer + area && labels.any?("devops::#{area}") if kind == :reviewer else capabilities(project).include?("#{kind} #{category}") end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 40bda3410e1..75d9a2d55b9 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -60,7 +60,8 @@ module Gitlab # rubocop:disable Metrics/ParameterLists def build( project:, user:, ref:, oldrev: nil, newrev: nil, - commits: [], commits_count: nil, message: nil, push_options: {}) + commits: [], commits_count: nil, message: nil, push_options: {}, + with_changed_files: true) commits = Array(commits) @@ -75,7 +76,7 @@ module Gitlab # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/38259 commit_attrs = Gitlab::GitalyClient.allow_n_plus_1_calls do commits_limited.map do |commit| - commit.hook_attrs(with_changed_files: true) + commit.hook_attrs(with_changed_files: with_changed_files) end end @@ -128,8 +129,6 @@ module Gitlab SAMPLE_DATA end - private - def checkout_sha(repository, newrev, ref) # Checkout sha is nil when we remove branch or tag return if Gitlab::Git.blank_ref?(newrev) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 34c1e6ad8ca..a12bbededc4 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -4,13 +4,13 @@ module Gitlab module Database include Gitlab::Metrics::Methods - # The max value of INTEGER type is the same between MySQL and PostgreSQL: # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html - # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html MAX_INT_VALUE = 2147483647 + # The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz: # https://www.postgresql.org/docs/9.1/static/datatype-datetime.html # https://dev.mysql.com/doc/refman/5.7/en/datetime.html + # FIXME: this should just be the max value of timestampz MAX_TIMESTAMP_VALUE = Time.at((1 << 31) - 1).freeze # Minimum schema version from which migrations are supported @@ -39,13 +39,14 @@ module Gitlab end def self.human_adapter_name - postgresql? ? 'PostgreSQL' : 'MySQL' - end - - def self.mysql? - adapter_name.casecmp('mysql2').zero? + if postgresql? + 'PostgreSQL' + else + 'Unknown' + end end + # @deprecated def self.postgresql? adapter_name.casecmp('postgresql').zero? end @@ -60,15 +61,14 @@ module Gitlab # Check whether the underlying database is in read-only mode def self.db_read_only? - if postgresql? - pg_is_in_recovery = - ActiveRecord::Base.connection.execute('SELECT pg_is_in_recovery()') - .first.fetch('pg_is_in_recovery') + pg_is_in_recovery = + ActiveRecord::Base + .connection + .execute('SELECT pg_is_in_recovery()') + .first + .fetch('pg_is_in_recovery') - Gitlab::Utils.to_boolean(pg_is_in_recovery) - else - false - end + Gitlab::Utils.to_boolean(pg_is_in_recovery) end def self.db_read_write? @@ -80,19 +80,19 @@ module Gitlab end def self.postgresql_9_or_less? - postgresql? && version.to_f < 10 + version.to_f < 10 end def self.join_lateral_supported? - postgresql? && version.to_f >= 9.3 + version.to_f >= 9.3 end def self.replication_slots_supported? - postgresql? && version.to_f >= 9.4 + version.to_f >= 9.4 end def self.postgresql_minimum_supported_version? - postgresql? && version.to_f >= 9.6 + version.to_f >= 9.6 end # map some of the function names that changed between PostgreSQL 9 and 10 @@ -118,51 +118,23 @@ module Gitlab end def self.nulls_last_order(field, direction = 'ASC') - order = "#{field} #{direction}" - - if postgresql? - order = "#{order} NULLS LAST" - else - # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL - # columns. In the (default) ascending order, `0` comes first. - order = "#{field} IS NULL, #{order}" if direction == 'ASC' - end - - order + Arel.sql("#{field} #{direction} NULLS LAST") end def self.nulls_first_order(field, direction = 'ASC') - order = "#{field} #{direction}" - - if postgresql? - order = "#{order} NULLS FIRST" - else - # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL - # columns. In the (default) ascending order, `0` comes first. - order = "#{field} IS NULL, #{order}" if direction == 'DESC' - end - - order + Arel.sql("#{field} #{direction} NULLS FIRST") end def self.random - postgresql? ? "RANDOM()" : "RAND()" + "RANDOM()" end def self.true_value - if postgresql? - "'t'" - else - 1 - end + "'t'" end def self.false_value - if postgresql? - "'f'" - else - 0 - end + "'f'" end def self.with_connection_pool(pool_size) @@ -182,7 +154,7 @@ module Gitlab # rows - An Array of Hash instances, each mapping the columns to their # values. # return_ids - When set to true the return value will be an Array of IDs of - # the inserted rows, this only works on PostgreSQL. + # the inserted rows # disable_quote - A key or an Array of keys to exclude from quoting (You # become responsible for protection from SQL injection for # these keys!) @@ -191,7 +163,6 @@ module Gitlab keys = rows.first.keys columns = keys.map { |key| connection.quote_column_name(key) } - return_ids = false if mysql? disable_quote = Array(disable_quote).to_set tuples = rows.map do |row| @@ -224,13 +195,14 @@ module Gitlab # pool_size - The size of the DB pool. # host - An optional host name to use instead of the default one. - def self.create_connection_pool(pool_size, host = nil) + def self.create_connection_pool(pool_size, host = nil, port = nil) # See activerecord-4.2.7.1/lib/active_record/connection_adapters/connection_specification.rb env = Rails.env original_config = ActiveRecord::Base.configurations env_config = original_config[env].merge('pool' => pool_size) env_config['host'] = host if host + env_config['port'] = port if port config = original_config.merge(env => env_config) @@ -258,11 +230,7 @@ module Gitlab def self.database_version row = connection.execute("SELECT VERSION()").first - if postgresql? - row['version'] - else - row.first - end + row['version'] end private_class_method :database_version @@ -284,17 +252,28 @@ module Gitlab end # inside_transaction? will return true if the caller is running within a transaction. Handles special cases - # when running inside a test environment, in which the entire test is running with a DatabaseCleaner transaction + # when running inside a test environment, where tests may be wrapped in transactions def self.inside_transaction? - ActiveRecord::Base.connection.open_transactions > open_transactions_baseline + if Rails.env.test? + ActiveRecord::Base.connection.open_transactions > open_transactions_baseline + else + ActiveRecord::Base.connection.open_transactions > 0 + end end - def self.open_transactions_baseline - if ::Rails.env.test? - return DatabaseCleaner.connections.count { |conn| conn.strategy.is_a?(DatabaseCleaner::ActiveRecord::Transaction) } - end + # These methods that access @open_transactions_baseline are not thread-safe. + # These are fine though because we only call these in RSpec's main thread. If we decide to run + # specs multi-threaded, we would need to use something like ThreadGroup to keep track of this value + def self.set_open_transactions_baseline + @open_transactions_baseline = ActiveRecord::Base.connection.open_transactions + end + + def self.reset_open_transactions_baseline + @open_transactions_baseline = 0 + end - 0 + def self.open_transactions_baseline + @open_transactions_baseline ||= 0 end private_class_method :open_transactions_baseline @@ -310,7 +289,7 @@ module Gitlab gitlab_database_transaction_seconds.observe(labels, duration_seconds) rescue Prometheus::Client::LabelSetValidator::LabelSetError => err # Ensure that errors in recording these metrics don't affect the operation of the application - Rails.logger.error("Unable to observe database transaction duration: #{err}") + Rails.logger.error("Unable to observe database transaction duration: #{err}") # rubocop:disable Gitlab/RailsLogger end # MonkeyPatch for ActiveRecord::Base for adding observability diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index f3d37ccd72a..eac61254bdf 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -37,16 +37,14 @@ module Gitlab # @return [Hash] of Model -> count mapping def self.approximate_counts(models, strategies: [TablesampleCountStrategy, ReltuplesCountStrategy, ExactCountStrategy]) strategies.each_with_object({}) do |strategy, counts_by_model| - if strategy.enabled? - models_with_missing_counts = models - counts_by_model.keys + models_with_missing_counts = models - counts_by_model.keys - break counts_by_model if models_with_missing_counts.empty? + break counts_by_model if models_with_missing_counts.empty? - counts = strategy.new(models_with_missing_counts).count + counts = strategy.new(models_with_missing_counts).count - counts.each do |model, count| - counts_by_model[model] = count - end + counts.each do |model, count| + counts_by_model[model] = count end end end diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb index fa6951eda22..0b8fe640bf8 100644 --- a/lib/gitlab/database/count/exact_count_strategy.rb +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -23,10 +23,6 @@ module Gitlab rescue *CONNECTION_ERRORS {} end - - def self.enabled? - true - end end end end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 695f6fa766e..6cd90c01ab2 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -31,10 +31,6 @@ module Gitlab {} end - def self.enabled? - Gitlab::Database.postgresql? - end - private # Models using single-type inheritance (STI) don't work with diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb index 7777f31f702..e9387a91a14 100644 --- a/lib/gitlab/database/count/tablesample_count_strategy.rb +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -28,10 +28,6 @@ module Gitlab {} end - def self.enabled? - Gitlab::Database.postgresql? && Feature.enabled?(:tablesample_counts) - end - private def perform_count(model, estimate) diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb index 79d2caff151..1392b397012 100644 --- a/lib/gitlab/database/date_time.rb +++ b/lib/gitlab/database/date_time.rb @@ -7,8 +7,7 @@ module Gitlab # the first of the `start_time_attrs` that isn't NULL. `SELECT` the resulting interval # along with an alias specified by the `as` parameter. # - # Note: For MySQL, the interval is returned in seconds. - # For PostgreSQL, the interval is returned as an INTERVAL type. + # Note: the interval is returned as an INTERVAL type. def subtract_datetimes(query_so_far, start_time_attrs, end_time_attrs, as) diff_fn = subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) @@ -16,17 +15,10 @@ module Gitlab end def subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) - if Gitlab::Database.postgresql? - Arel::Nodes::Subtraction.new( - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) - elsif Gitlab::Database.mysql? - Arel::Nodes::NamedFunction.new( - "TIMESTAMPDIFF", - [Arel.sql('second'), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) - end + Arel::Nodes::Subtraction.new( + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)) + ) end end end diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index 862ab96c887..1f47f320a29 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -6,47 +6,25 @@ module Gitlab class Grant < ActiveRecord::Base include FromUnion - self.table_name = - if Database.postgresql? - 'information_schema.role_table_grants' - else - 'information_schema.schema_privileges' - end + self.table_name = 'information_schema.role_table_grants' # Returns true if the current user can create and execute triggers on the # given table. def self.create_and_execute_trigger?(table) - if Database.postgresql? - # We _must not_ use quote_table_name as this will produce double - # quotes on PostgreSQL and for "has_table_privilege" we need single - # quotes. - quoted_table = connection.quote(table) - - begin - from(nil) - .pluck("has_table_privilege(#{quoted_table}, 'TRIGGER')") - .first - rescue ActiveRecord::StatementInvalid - # This error is raised when using a non-existing table name. In this - # case we just want to return false as a user technically can't - # create triggers for such a table. - false - end - else - queries = [ - Grant.select(1) - .from('information_schema.user_privileges') - .where("PRIVILEGE_TYPE = 'SUPER'") - .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"), - - Grant.select(1) - .from('information_schema.schema_privileges') - .where("PRIVILEGE_TYPE = 'TRIGGER'") - .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name) - .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") - ] + # We _must not_ use quote_table_name as this will produce double + # quotes on PostgreSQL and for "has_table_privilege" we need single + # quotes. + quoted_table = connection.quote(table) - Grant.from_union(queries, alias_as: 'privs').any? + begin + from(nil) + .pluck(Arel.sql("has_table_privilege(#{quoted_table}, 'TRIGGER')")) + .first + rescue ActiveRecord::StatementInvalid + # This error is raised when using a non-existing table name. In this + # case we just want to return false as a user technically can't + # create triggers for such a table. + false end end end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index b8d895dee7d..603b125d8b4 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -17,13 +17,9 @@ module Gitlab def extract_median(results) result = results.compact.first - if Gitlab::Database.postgresql? - result = result.first.presence + result = result.first.presence - result['median']&.to_f if result - elsif Gitlab::Database.mysql? - result.to_a.flatten.first - end + result['median']&.to_f if result end def extract_medians(results) @@ -34,31 +30,6 @@ module Gitlab end end - def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - query = arel_table.from - .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) - .project(average([arel_table[column_sym]], 'median')) - .where( - Arel::Nodes::Between.new( - Arel.sql("(select @row_id := @row_id + 1)"), - Arel::Nodes::And.new( - [Arel.sql('@ct/2.0'), - Arel.sql('@ct/2.0 + 1')] - ) - ) - ). - # Disallow negative values - where(arel_table[column_sym].gteq(0)) - - [ - Arel.sql("CREATE TEMPORARY TABLE IF NOT EXISTS #{query_so_far.to_sql}"), - Arel.sql("set @ct := (select count(1) from #{arel_table.table_name});"), - Arel.sql("set @row_id := 0;"), - query.to_sql, - Arel.sql("DROP TEMPORARY TABLE IF EXISTS #{arel_table.table_name};") - ] - end - def pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column = nil) # Create a CTE with the column we're operating on, row number (after sorting by the column # we're operating on), and count of the table we're operating on (duplicated across) all rows @@ -113,18 +84,8 @@ module Gitlab private - def median_queries(arel_table, query_so_far, column_sym, partition_column = nil) - if Gitlab::Database.postgresql? - pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) - elsif Gitlab::Database.mysql? - raise NotSupportedError, "partition_column is not supported for MySQL" if partition_column - - mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - end - end - def execute_queries(arel_table, query_so_far, column_sym, partition_column = nil) - queries = median_queries(arel_table, query_so_far, column_sym, partition_column) + queries = pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) Array.wrap(queries).map { |query| ActiveRecord::Base.connection.execute(query) } end @@ -176,8 +137,6 @@ module Gitlab end def extract_diff_epoch(diff) - return diff unless Gitlab::Database.postgresql? - Arel.sql(%Q{EXTRACT(EPOCH FROM (#{diff.to_sql}))}) end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index e2cbf91f281..57a413f8e04 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -6,31 +6,45 @@ module Gitlab BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time + PERMITTED_TIMESTAMP_COLUMNS = %i[created_at updated_at deleted_at].to_set.freeze + DEFAULT_TIMESTAMP_COLUMNS = %i[created_at updated_at].freeze + # Adds `created_at` and `updated_at` columns with timezone information. # # This method is an improved version of Rails' built-in method `add_timestamps`. # + # By default, adds `created_at` and `updated_at` columns, but these can be specified as: + # + # add_timestamps_with_timezone(:my_table, columns: [:created_at, :deleted_at]) + # + # This allows you to create just the timestamps you need, saving space. + # # Available options are: - # default - The default value for the column. - # null - When set to `true` the column will allow NULL values. + # :default - The default value for the column. + # :null - When set to `true` the column will allow NULL values. # The default is to not allow NULL values. + # :columns - the column names to create. Must be one + # of `Gitlab::Database::MigrationHelpers::PERMITTED_TIMESTAMP_COLUMNS`. + # Default value: `DEFAULT_TIMESTAMP_COLUMNS` + # + # All options are optional. def add_timestamps_with_timezone(table_name, options = {}) options[:null] = false if options[:null].nil? + columns = options.fetch(:columns, DEFAULT_TIMESTAMP_COLUMNS) + default_value = options[:default] - [:created_at, :updated_at].each do |column_name| - if options[:default] && transaction_open? - raise '`add_timestamps_with_timezone` with default value cannot be run inside a transaction. ' \ - 'You can disable transactions by calling `disable_ddl_transaction!` ' \ - 'in the body of your migration class' - end + validate_not_in_transaction!(:add_timestamps_with_timezone, 'with default value') if default_value + + columns.each do |column_name| + validate_timestamp_column_name!(column_name) # If default value is presented, use `add_column_with_default` method instead. - if options[:default] + if default_value add_column_with_default( table_name, column_name, :datetime_with_timezone, - default: options[:default], + default: default_value, allow_null: options[:null] ) else @@ -39,10 +53,22 @@ module Gitlab end end - # Creates a new index, concurrently when supported + # To be used in the `#down` method of migrations that + # use `#add_timestamps_with_timezone`. # - # On PostgreSQL this method creates an index concurrently, on MySQL this - # creates a regular index. + # Available options are: + # :columns - the column names to remove. Must be one + # Default value: `DEFAULT_TIMESTAMP_COLUMNS` + # + # All options are optional. + def remove_timestamps(table_name, options = {}) + columns = options.fetch(:columns, DEFAULT_TIMESTAMP_COLUMNS) + columns.each do |column_name| + remove_column(table_name, column_name) + end + end + + # Creates a new index, concurrently # # Example: # @@ -56,12 +82,10 @@ module Gitlab 'in the body of your migration class' end - if Database.postgresql? - options = options.merge({ algorithm: :concurrently }) - end + options = options.merge({ algorithm: :concurrently }) if index_exists?(table_name, column_name, options) - Rails.logger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" + Rails.logger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" # rubocop:disable Gitlab/RailsLogger return end @@ -70,9 +94,7 @@ module Gitlab end end - # Removes an existed index, concurrently when supported - # - # On PostgreSQL this method removes an index concurrently. + # Removes an existed index, concurrently # # Example: # @@ -91,7 +113,7 @@ module Gitlab end unless index_exists?(table_name, column_name, options) - Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" + Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" # rubocop:disable Gitlab/RailsLogger return end @@ -100,9 +122,7 @@ module Gitlab end end - # Removes an existing index, concurrently when supported - # - # On PostgreSQL this method removes an index concurrently. + # Removes an existing index, concurrently # # Example: # @@ -121,7 +141,7 @@ module Gitlab end unless index_exists_by_name?(table_name, index_name) - Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, index_name: #{index_name}" + Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, index_name: #{index_name}" # rubocop:disable Gitlab/RailsLogger return end @@ -132,8 +152,6 @@ module Gitlab # Only available on Postgresql >= 9.2 def supports_drop_index_concurrently? - return false unless Database.postgresql? - version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i version >= 90200 @@ -141,14 +159,15 @@ module Gitlab # Adds a foreign key with only minimal locking on the tables involved. # - # This method only requires minimal locking when using PostgreSQL. When - # using MySQL this method will use Rails' default `add_foreign_key`. + # This method only requires minimal locking # # source - The source table containing the foreign key. # target - The target table the key points to. # column - The name of the column to create the foreign key on. # on_delete - The action to perform when associated data is removed, # defaults to "CASCADE". + # + # rubocop:disable Gitlab/RailsLogger def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, name: nil) # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. @@ -156,27 +175,7 @@ module Gitlab raise 'add_concurrent_foreign_key can not be run inside a transaction' end - # While MySQL does allow disabling of foreign keys it has no equivalent - # of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall - # back to the normal foreign key procedure. - if Database.mysql? - if foreign_key_exists?(source, target, column: column) - Rails.logger.warn "Foreign key not created because it exists already " \ - "(this may be due to an aborted migration or similar): " \ - "source: #{source}, target: #{target}, column: #{column}" - return - end - - key_options = { column: column, on_delete: on_delete } - - # The MySQL adapter tries to create a foreign key without a name when - # `:name` is nil, instead of generating a name for us. - key_options[:name] = name if name - - return add_foreign_key(source, target, key_options) - else - on_delete = 'SET NULL' if on_delete == :nullify - end + on_delete = 'SET NULL' if on_delete == :nullify key_name = name || concurrent_foreign_key_name(source, column) @@ -208,6 +207,7 @@ module Gitlab execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};") end end + # rubocop:enable Gitlab/RailsLogger def foreign_key_exists?(source, target = nil, column: nil) foreign_keys(source).any? do |key| @@ -233,7 +233,7 @@ module Gitlab # Long-running migrations may take more than the timeout allowed by # the database. Disable the session's statement timeout to ensure - # migrations don't get killed prematurely. (PostgreSQL only) + # migrations don't get killed prematurely. # # There are two possible ways to disable the statement timeout: # @@ -245,15 +245,6 @@ module Gitlab # otherwise the statement will still be disabled until connection is dropped # or `RESET ALL` is executed def disable_statement_timeout - # bypass disabled_statement logic when not using postgres, but still execute block when one is given - unless Database.postgresql? - if block_given? - yield - end - - return - end - if block_given? begin execute('SET statement_timeout TO 0') @@ -479,7 +470,7 @@ module Gitlab # We set the default value _after_ adding the column so we don't end up # updating any existing data with the default value. This isn't # necessary since we copy over old values further down. - change_column_default(table, new, old_col.default) if old_col.default + change_column_default(table, new, old_col.default) unless old_col.default.nil? install_rename_triggers(table, old, new) @@ -491,6 +482,16 @@ module Gitlab copy_foreign_keys(table, old, new) end + def undo_rename_column_concurrently(table, old, new) + trigger_name = rename_trigger_name(table, old, new) + + check_trigger_permissions!(table) + + remove_rename_triggers_for_postgresql(table, trigger_name) + + remove_column(table, new) + end + # Installs triggers in a table that keep a new column in sync with an old # one. # @@ -503,13 +504,12 @@ module Gitlab quoted_old = quote_column_name(old_column) quoted_new = quote_column_name(new_column) - if Database.postgresql? - install_rename_triggers_for_postgresql(trigger_name, quoted_table, - quoted_old, quoted_new) - else - install_rename_triggers_for_mysql(trigger_name, quoted_table, - quoted_old, quoted_new) - end + install_rename_triggers_for_postgresql( + trigger_name, + quoted_table, + quoted_old, + quoted_new + ) end # Changes the type of a column concurrently. @@ -552,15 +552,40 @@ module Gitlab check_trigger_permissions!(table) - if Database.postgresql? - remove_rename_triggers_for_postgresql(table, trigger_name) - else - remove_rename_triggers_for_mysql(trigger_name) - end + remove_rename_triggers_for_postgresql(table, trigger_name) remove_column(table, old) end + def undo_cleanup_concurrent_column_rename(table, old, new, type: nil) + if transaction_open? + raise 'undo_cleanup_concurrent_column_rename can not be run inside a transaction' + end + + check_trigger_permissions!(table) + + new_column = column_for(table, new) + + add_column(table, old, type || new_column.type, + limit: new_column.limit, + precision: new_column.precision, + scale: new_column.scale) + + # We set the default value _after_ adding the column so we don't end up + # updating any existing data with the default value. This isn't + # necessary since we copy over old values further down. + change_column_default(table, old, new_column.default) unless new_column.default.nil? + + install_rename_triggers(table, old, new) + + update_column_in_batches(table, old, Arel::Table.new(table)[new]) + + change_column_null(table, old, false) unless new_column.null + + copy_indexes(table, new, old) + copy_foreign_keys(table, new, old) + end + # Changes the column type of a table using a background migration. # # Because this method uses a background migration it's more suitable for @@ -761,31 +786,16 @@ module Gitlab EOF execute <<-EOF.strip_heredoc - CREATE TRIGGER #{trigger} - BEFORE INSERT OR UPDATE - ON #{table} - FOR EACH ROW - EXECUTE PROCEDURE #{trigger}() - EOF - end - - # Installs the triggers necessary to perform a concurrent column rename on - # MySQL. - def install_rename_triggers_for_mysql(trigger, table, old, new) - execute <<-EOF.strip_heredoc - CREATE TRIGGER #{trigger}_insert - BEFORE INSERT + DROP TRIGGER IF EXISTS #{trigger} ON #{table} - FOR EACH ROW - SET NEW.#{new} = NEW.#{old} EOF execute <<-EOF.strip_heredoc - CREATE TRIGGER #{trigger}_update - BEFORE UPDATE + CREATE TRIGGER #{trigger} + BEFORE INSERT OR UPDATE ON #{table} FOR EACH ROW - SET NEW.#{new} = NEW.#{old} + EXECUTE PROCEDURE #{trigger}() EOF end @@ -795,12 +805,6 @@ module Gitlab execute("DROP FUNCTION IF EXISTS #{trigger}()") end - # Removes the triggers used for renaming a MySQL column concurrently. - def remove_rename_triggers_for_mysql(trigger) - execute("DROP TRIGGER IF EXISTS #{trigger}_insert") - execute("DROP TRIGGER IF EXISTS #{trigger}_update") - end - # Returns the (base) name to use for triggers when renaming columns. def rename_trigger_name(table, old, new) 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12) @@ -850,8 +854,6 @@ module Gitlab order: index.orders } - # These options are not supported by MySQL, so we only add them if - # they were previously set. options[:using] = index.using if index.using options[:where] = index.where if index.where @@ -891,26 +893,16 @@ module Gitlab end # This will replace the first occurrence of a string in a column with - # the replacement - # On postgresql we can use `regexp_replace` for that. - # On mysql we find the location of the pattern, and overwrite it - # with the replacement + # the replacement using `regexp_replace` def replace_sql(column, pattern, replacement) quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s) quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) - if Database.mysql? - locate = Arel::Nodes::NamedFunction - .new('locate', [quoted_pattern, column]) - insert_in_place = Arel::Nodes::NamedFunction - .new('insert', [column, locate, pattern.size, quoted_replacement]) + replace = Arel::Nodes::NamedFunction.new( + "regexp_replace", [column, quoted_pattern, quoted_replacement] + ) - Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) - else - replace = Arel::Nodes::NamedFunction - .new("regexp_replace", [column, quoted_pattern, quoted_replacement]) - Arel::Nodes::SqlLiteral.new(replace.to_sql) - end + Arel::Nodes::SqlLiteral.new(replace.to_sql) end def remove_foreign_key_if_exists(*args) @@ -952,11 +944,7 @@ database (#{dbname}) using a super user and running: ALTER #{user} WITH SUPERUSER -For MySQL you instead need to run: - - GRANT ALL PRIVILEGES ON #{dbname}.* TO #{user}@'%' - -Both queries will grant the user super user permissions, ensuring you don't run +This query will grant the user super user permissions, ensuring you don't run into similar problems in the future (e.g. when new tables are created). EOF end @@ -1059,10 +1047,6 @@ into similar problems in the future (e.g. when new tables are created). # This will include indexes using an expression on the column, for example: # `CREATE INDEX CONCURRENTLY index_name ON table (LOWER(column));` # - # For mysql, it falls back to the default ActiveRecord implementation that - # will not find custom indexes. But it will select by name without passing - # a column. - # # We can remove this when upgrading to Rails 5 with an updated `index_exists?`: # - https://github.com/rails/rails/commit/edc2b7718725016e988089b5fb6d6fb9d6e16882 # @@ -1073,10 +1057,8 @@ into similar problems in the future (e.g. when new tables are created). # does not find indexes without passing a column name. if indexes(table).map(&:name).include?(index.to_s) true - elsif Gitlab::Database.postgresql? - postgres_exists_by_name?(table, index) else - false + postgres_exists_by_name?(table, index) end end @@ -1092,8 +1074,26 @@ into similar problems in the future (e.g. when new tables are created). connection.select_value(index_sql).to_i > 0 end - def mysql_compatible_index_length - Gitlab::Database.mysql? ? 20 : nil + private + + def validate_timestamp_column_name!(column_name) + return if PERMITTED_TIMESTAMP_COLUMNS.member?(column_name) + + raise <<~MESSAGE + Illegal timestamp column name! Got #{column_name}. + Must be one of: #{PERMITTED_TIMESTAMP_COLUMNS.to_a} + MESSAGE + end + + def validate_not_in_transaction!(method_name, modifier = nil) + return unless transaction_open? + + raise <<~ERROR + #{["`#{method_name}`", modifier].compact.join(' ')} cannot be run inside a transaction. + + You can disable transactions by calling `disable_ddl_transaction!` in the body of + your migration class + ERROR end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 60afa4bcd52..565f34b78b7 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -51,14 +51,10 @@ module Gitlab quoted_old_full_path = quote_string(old_full_path) quoted_old_wildcard_path = quote_string("#{old_full_path}/%") - filter = if Database.mysql? - "lower(routes.path) = lower('#{quoted_old_full_path}') "\ - "OR routes.path LIKE '#{quoted_old_wildcard_path}'" - else - "routes.id IN "\ - "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ - "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" - end + filter = + "routes.id IN "\ + "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ + "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" replace_statement = replace_sql(Route.arel_table[:path], old_full_path, diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 6bbad707f0f..3e8a9b89998 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -70,7 +70,7 @@ module Gitlab unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path) message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}" - Rails.logger.error message + Rails.logger.error message # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 580be9fe267..4dc7a62797a 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -56,7 +56,7 @@ module Gitlab unless gitlab_shell.mv_repository(project.repository_storage, old_path, new_path) - Rails.logger.error "Error moving #{old_path} to #{new_path}" + Rails.logger.error "Error moving #{old_path} to #{new_path}" # rubocop:disable Gitlab/RailsLogger end end diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb index 109ae7893da..ddbabc9098e 100644 --- a/lib/gitlab/database/sha_attribute.rb +++ b/lib/gitlab/database/sha_attribute.rb @@ -2,14 +2,9 @@ module Gitlab module Database - BINARY_TYPE = - if Gitlab::Database.postgresql? - # PostgreSQL defines its own class with slightly different - # behaviour from the default Binary type. - ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea - else - ActiveModel::Type::Binary - end + # PostgreSQL defines its own class with slightly different + # behaviour from the default Binary type. + BINARY_TYPE = ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa). # diff --git a/lib/gitlab/database/subquery.rb b/lib/gitlab/database/subquery.rb index 10971d2b274..2a6f39c6a27 100644 --- a/lib/gitlab/database/subquery.rb +++ b/lib/gitlab/database/subquery.rb @@ -6,11 +6,7 @@ module Gitlab class << self def self_join(relation) t = relation.arel_table - # Work around a bug in Rails 5, where LIMIT causes trouble - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/51729 - r = relation.limit(nil).arel - r.take(relation.limit_value) if relation.limit_value - t2 = r.as('t2') + t2 = relation.arel.as('t2') relation.unscoped.joins(t.join(t2).on(t[:id].eq(t2[:id])).join_sources.first) end diff --git a/lib/gitlab/database_importers/common_metrics.rb b/lib/gitlab/database_importers/common_metrics.rb new file mode 100644 index 00000000000..f964ae8a275 --- /dev/null +++ b/lib/gitlab/database_importers/common_metrics.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module CommonMetrics + end + end +end diff --git a/lib/gitlab/database_importers/common_metrics/importer.rb b/lib/gitlab/database_importers/common_metrics/importer.rb new file mode 100644 index 00000000000..6c61e05674e --- /dev/null +++ b/lib/gitlab/database_importers/common_metrics/importer.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module CommonMetrics + class Importer + MissingQueryId = Class.new(StandardError) + + attr_reader :content + + def initialize(filename = 'common_metrics.yml') + @content = YAML.load_file(Rails.root.join('config', 'prometheus', filename)) + end + + def execute + CommonMetrics::PrometheusMetric.reset_column_information + + process_content do |id, attributes| + find_or_build_metric!(id) + .update!(**attributes) + end + end + + private + + def process_content(&blk) + content['panel_groups'].map do |group| + process_group(group, &blk) + end + end + + def process_group(group, &blk) + attributes = { + group: find_group_title_key(group['group']) + } + + group['panels'].map do |panel| + process_panel(panel, attributes, &blk) + end + end + + def process_panel(panel, attributes, &blk) + attributes = attributes.merge( + title: panel['title'], + y_label: panel['y_label']) + + panel['metrics'].map do |metric_details| + process_metric_details(metric_details, attributes, &blk) + end + end + + def process_metric_details(metric_details, attributes, &blk) + attributes = attributes.merge( + legend: metric_details['label'], + query: metric_details['query_range'], + unit: metric_details['unit']) + + yield(metric_details['id'], attributes) + end + + def find_or_build_metric!(id) + raise MissingQueryId unless id + + CommonMetrics::PrometheusMetric.common.find_by(identifier: id) || + CommonMetrics::PrometheusMetric.new(common: true, identifier: id) + end + + def find_group_title_key(title) + CommonMetrics::PrometheusMetricEnums.groups[find_group_title(title)] + end + + def find_group_title(title) + CommonMetrics::PrometheusMetricEnums.group_titles.invert[title] + end + end + end + end +end diff --git a/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb b/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb new file mode 100644 index 00000000000..b4a392cbea9 --- /dev/null +++ b/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module CommonMetrics + class PrometheusMetric < ApplicationRecord + enum group: PrometheusMetricEnums.groups + scope :common, -> { where(common: true) } + end + end + end +end diff --git a/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb new file mode 100644 index 00000000000..c9e957ec7c0 --- /dev/null +++ b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module CommonMetrics + module PrometheusMetricEnums + def self.groups + { + # built-in groups + nginx_ingress_vts: -1, + ha_proxy: -2, + aws_elb: -3, + nginx: -4, + kubernetes: -5, + nginx_ingress: -6, + + # custom groups + business: 0, + response: 1, + system: 2 + } + end + + def self.group_titles + { + business: _('Business metrics (Custom)'), + response: _('Response metrics (Custom)'), + system: _('System metrics (Custom)'), + nginx_ingress_vts: _('Response metrics (NGINX Ingress VTS)'), + nginx_ingress: _('Response metrics (NGINX Ingress)'), + ha_proxy: _('Response metrics (HA Proxy)'), + aws_elb: _('Response metrics (AWS ELB)'), + nginx: _('Response metrics (NGINX)'), + kubernetes: _('System metrics (Kubernetes)') + } + end + end + end + end +end diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb new file mode 100644 index 00000000000..3a170e8b5f8 --- /dev/null +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -0,0 +1,249 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module SelfMonitoring + module Project + class CreateService < ::BaseService + include Stepable + + STEPS_ALLOWED_TO_FAIL = [ + :validate_application_settings, :validate_project_created, :validate_admins + ].freeze + + VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL + PROJECT_NAME = 'GitLab Instance Administration' + + steps :validate_application_settings, + :validate_project_created, + :validate_admins, + :create_group, + :create_project, + :save_project_id, + :add_group_members, + :add_to_whitelist, + :add_prometheus_manual_configuration + + def initialize + super(nil) + end + + def execute! + result = execute_steps + + if result[:status] == :success + result + elsif STEPS_ALLOWED_TO_FAIL.include?(result[:failed_step]) + success + else + raise StandardError, result[:message] + end + end + + private + + def validate_application_settings + return success if application_settings + + log_error(_('No application_settings found')) + error(_('No application_settings found')) + end + + def validate_project_created + return success unless project_created? + + log_error(_('Project already created')) + error(_('Project already created')) + end + + def validate_admins + unless instance_admins.any? + log_error(_('No active admin user found')) + return error(_('No active admin user found')) + end + + success + end + + def create_group + if project_created? + log_info(_('Instance administrators group already exists')) + @group = application_settings.instance_administration_project.owner + return success(group: @group) + end + + @group = ::Groups::CreateService.new(group_owner, create_group_params).execute + + if @group.persisted? + success(group: @group) + else + error(_('Could not create group')) + end + end + + def create_project + if project_created? + log_info(_('Instance administration project already exists')) + @project = application_settings.instance_administration_project + return success(project: project) + end + + @project = ::Projects::CreateService.new(group_owner, create_project_params).execute + + if project.persisted? + success(project: project) + else + log_error(_("Could not create instance administration project. Errors: %{errors}") % { errors: project.errors.full_messages }) + error(_('Could not create project')) + end + end + + def save_project_id + return success if project_created? + + result = application_settings.update(instance_administration_project_id: @project.id) + + if result + success + else + log_error(_("Could not save instance administration project ID, errors: %{errors}") % { errors: application_settings.errors.full_messages }) + error(_('Could not save project ID')) + end + end + + def add_group_members + members = @group.add_users(members_to_add, Gitlab::Access::MAINTAINER) + errors = members.flat_map { |member| member.errors.full_messages } + + if errors.any? + log_error(_('Could not add admins as members to self-monitoring project. Errors: %{errors}') % { errors: errors }) + error(_('Could not add admins as members')) + else + success + end + end + + def add_to_whitelist + return success unless prometheus_enabled? + return success unless prometheus_listen_address.present? + + uri = parse_url(internal_prometheus_listen_address_uri) + return error(_('Prometheus listen_address is not a valid URI')) unless uri + + application_settings.add_to_outbound_local_requests_whitelist([uri.normalized_host]) + result = application_settings.save + + if result + # Expire the Gitlab::CurrentSettings cache after updating the whitelist. + # This happens automatically in an after_commit hook, but in migrations, + # the after_commit hook only runs at the end of the migration. + Gitlab::CurrentSettings.expire_current_application_settings + success + else + log_error(_("Could not add prometheus URL to whitelist, errors: %{errors}") % { errors: application_settings.errors.full_messages }) + error(_('Could not add prometheus URL to whitelist')) + end + end + + def add_prometheus_manual_configuration + return success unless prometheus_enabled? + return success unless prometheus_listen_address.present? + + service = project.find_or_initialize_service('prometheus') + + unless service.update(prometheus_service_attributes) + log_error(_('Could not save prometheus manual configuration for self-monitoring project. Errors: %{errors}') % { errors: service.errors.full_messages }) + return error(_('Could not save prometheus manual configuration')) + end + + success + end + + def application_settings + @application_settings ||= ApplicationSetting.current_without_cache + end + + def project_created? + application_settings.instance_administration_project.present? + end + + def parse_url(uri_string) + Addressable::URI.parse(uri_string) + rescue Addressable::URI::InvalidURIError, TypeError + end + + def prometheus_enabled? + Gitlab.config.prometheus.enable if Gitlab.config.prometheus + rescue Settingslogic::MissingSetting + log_error(_('prometheus.enable is not present in gitlab.yml')) + + false + end + + def prometheus_listen_address + Gitlab.config.prometheus.listen_address if Gitlab.config.prometheus + rescue Settingslogic::MissingSetting + log_error(_('prometheus.listen_address is not present in gitlab.yml')) + + nil + end + + def instance_admins + @instance_admins ||= User.admins.active + end + + def group_owner + instance_admins.first + end + + def members_to_add + # Exclude admins who are already members of group because + # `@group.add_users(users)` returns an error if the users parameter contains + # users who are already members of the group. + instance_admins - @group.members.collect(&:user) + end + + def create_group_params + { + name: 'GitLab Instance Administrators', + path: "gitlab-instance-administrators-#{SecureRandom.hex(4)}", + visibility_level: VISIBILITY_LEVEL + } + end + + def docs_path + Rails.application.routes.url_helpers.help_page_path( + 'administration/monitoring/gitlab_instance_administration_project/index' + ) + end + + def create_project_params + { + initialize_with_readme: true, + visibility_level: VISIBILITY_LEVEL, + name: PROJECT_NAME, + description: "This project is automatically generated and will be used to help monitor this GitLab instance. [More information](#{docs_path})", + namespace_id: @group.id + } + end + + def internal_prometheus_listen_address_uri + if prometheus_listen_address.starts_with?('http') + prometheus_listen_address + else + 'http://' + prometheus_listen_address + end + end + + def prometheus_service_attributes + { + api_url: internal_prometheus_listen_address_uri, + manual_configuration: true, + active: true + } + end + end + end + end + end +end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 01fd261404b..86e532766b1 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -7,7 +7,7 @@ module Gitlab CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') - IGNORED_FILES_REGEX = /VERSION|CHANGELOG\.md/i.freeze + IGNORED_FILES_REGEX = /VERSION|CHANGELOG\.md|doc\/.+/i.freeze PLEASE_READ_THIS_BANNER = %Q{ ============================================================ ===================== PLEASE READ THIS ===================== diff --git a/lib/gitlab/email/hook/disable_email_interceptor.rb b/lib/gitlab/email/hook/disable_email_interceptor.rb index 6b6b1d85109..58dc1527c7a 100644 --- a/lib/gitlab/email/hook/disable_email_interceptor.rb +++ b/lib/gitlab/email/hook/disable_email_interceptor.rb @@ -7,7 +7,7 @@ module Gitlab def self.delivering_email(message) message.perform_deliveries = false - Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" + Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/email/hook/smime_signature_interceptor.rb b/lib/gitlab/email/hook/smime_signature_interceptor.rb new file mode 100644 index 00000000000..e48041d9218 --- /dev/null +++ b/lib/gitlab/email/hook/smime_signature_interceptor.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Hook + class SmimeSignatureInterceptor + # Sign emails with SMIME if enabled + class << self + def delivering_email(message) + signed_message = Gitlab::Email::Smime::Signer.sign( + cert: certificate.cert, + key: certificate.key, + data: message.encoded) + signed_email = Mail.new(signed_message) + + overwrite_body(message, signed_email) + overwrite_headers(message, signed_email) + end + + private + + def certificate + @certificate ||= Gitlab::Email::Smime::Certificate.from_files(key_path, cert_path) + end + + def key_path + Gitlab.config.gitlab.email_smime.key_file + end + + def cert_path + Gitlab.config.gitlab.email_smime.cert_file + end + + def overwrite_body(message, signed_email) + # since this is a multipart email, assignment to nil is important, + # otherwise Message#body will add a new mail part + message.body = nil + message.body = signed_email.body.encoded + end + + def overwrite_headers(message, signed_email) + message.content_disposition = signed_email.content_disposition + message.content_transfer_encoding = signed_email.content_transfer_encoding + message.content_type = signed_email.content_type + end + end + end + end + end +end diff --git a/lib/gitlab/email/smime/certificate.rb b/lib/gitlab/email/smime/certificate.rb new file mode 100644 index 00000000000..b331c4ca19c --- /dev/null +++ b/lib/gitlab/email/smime/certificate.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Smime + class Certificate + include OpenSSL + + attr_reader :key, :cert + + def key_string + @key.to_s + end + + def cert_string + @cert.to_pem + end + + def self.from_strings(key_string, cert_string) + key = PKey::RSA.new(key_string) + cert = X509::Certificate.new(cert_string) + new(key, cert) + end + + def self.from_files(key_path, cert_path) + from_strings(File.read(key_path), File.read(cert_path)) + end + + def initialize(key, cert) + @key = key + @cert = cert + end + end + end + end +end diff --git a/lib/gitlab/email/smime/signer.rb b/lib/gitlab/email/smime/signer.rb new file mode 100644 index 00000000000..2fa83014003 --- /dev/null +++ b/lib/gitlab/email/smime/signer.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'openssl' + +module Gitlab + module Email + module Smime + # Tooling for signing and verifying data with SMIME + class Signer + include OpenSSL + + def self.sign(cert:, key:, data:) + signed_data = PKCS7.sign(cert, key, data, nil, PKCS7::DETACHED) + PKCS7.write_smime(signed_data) + end + + # return nil if data cannot be verified, otherwise the signed content data + def self.verify_signature(cert:, ca_cert: nil, signed_data:) + store = X509::Store.new + store.set_default_paths + store.add_cert(ca_cert) if ca_cert + + signed_smime = PKCS7.read_smime(signed_data) + signed_smime if signed_smime.verify([cert], store) + end + end + end + end +end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 5a61a7f5d60..88729babb2b 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -59,7 +59,7 @@ module Gitlab begin CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') rescue ArgumentError => e - Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") # rubocop:disable Gitlab/RailsLogger '' end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 17fbecbd097..d09dcdbb337 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -61,6 +61,10 @@ module Gitlab Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z), 'realtime_changes_import_gitea' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z), + 'merge_request_widget' ) ].freeze diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb index 7961d4bbd6e..61eb030563d 100644 --- a/lib/gitlab/exclusive_lease_helpers.rb +++ b/lib/gitlab/exclusive_lease_helpers.rb @@ -15,17 +15,18 @@ module Gitlab raise ArgumentError, 'Key needs to be specified' unless key lease = Gitlab::ExclusiveLease.new(key, timeout: ttl) + retried = false until uuid = lease.try_obtain # Keep trying until we obtain the lease. To prevent hammering Redis too # much we'll wait for a bit. sleep(sleep_sec) - break if (retries -= 1) < 0 + (retries -= 1) < 0 ? break : retried ||= true end raise FailedToObtainLockError, 'Failed to obtain a lock' unless uuid - yield + yield(retried) ensure Gitlab::ExclusiveLease.cancel(key, uuid) end diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index 3c71031a8d9..841f9de8d4a 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -20,7 +20,7 @@ module Gitlab path: repo.path, namespace: namespace, creator: current_user, - visibility_level: Gitlab::VisibilityLevel::INTERNAL, + visibility_level: Gitlab::VisibilityLevel::PRIVATE, import_type: 'fogbugz', import_source: repo.name, import_url: Project::UNKNOWN_IMPORT_URL, diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 2d1c9ac40ae..6b52d6e88e5 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -27,7 +27,15 @@ module Gitlab klass = target_parent.is_a?(Namespace) ? NamespaceFileUploader : FileUploader moved = klass.copy_to(file, target_parent) - moved.markdown_link + + moved_markdown = moved.markdown_link + + # Prevents rewrite of plain links as embedded + if was_embedded?(markdown) + moved_markdown + else + moved_markdown.sub(/\A!/, "") + end end end @@ -43,6 +51,10 @@ module Gitlab referenced_files.compact.select(&:exists?) end + def was_embedded?(markdown) + markdown.starts_with?("!") + end + private def find_file(project, secret, file) diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 44a62586a23..df9f33baec2 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -9,6 +9,7 @@ module Gitlab # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze BLANK_SHA = ('0' * 40).freeze + COMMIT_ID = /\A[0-9a-f]{40}\z/.freeze TAG_REF_PREFIX = "refs/tags/".freeze BRANCH_REF_PREFIX = "refs/heads/".freeze @@ -65,6 +66,10 @@ module Gitlab ref == BLANK_SHA end + def commit_id?(ref) + COMMIT_ID.match?(ref) + end + def version Gitlab::Git::Version.git_version end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 060a29be782..27032602828 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -55,6 +55,10 @@ module Gitlab @name = @relative_path.split("/").last end + def to_s + "<#{self.class.name}: #{self.gl_project_path}>" + end + def ==(other) other.is_a?(self.class) && [storage, relative_path] == [other.storage, other.relative_path] end @@ -464,6 +468,18 @@ module Gitlab end end + # Returns path to url mappings for submodules + # + # Ex. + # @repository.submodule_urls_for('master') + # # => { 'rack' => 'git@localhost:rack.git' } + # + def submodule_urls_for(ref) + wrapped_gitaly_errors do + gitaly_submodule_urls_for(ref) + end + end + # Return total commits count accessible from passed ref def commit_count(ref) wrapped_gitaly_errors do @@ -861,13 +877,13 @@ module Gitlab def multi_action( user, branch_name:, message:, actions:, author_email: nil, author_name: nil, - start_branch_name: nil, start_repository: self, + start_branch_name: nil, start_sha: nil, start_repository: self, force: false) wrapped_gitaly_errors do gitaly_operation_client.user_commit_files(user, branch_name, message, actions, author_email, author_name, - start_branch_name, start_repository, force) + start_branch_name, start_repository, force, start_sha) end end # rubocop:enable Metrics/ParameterLists @@ -936,7 +952,7 @@ module Gitlab gitaly_repository_client.cleanup if exists? end rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup - Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}") + Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}") # rubocop:disable Gitlab/RailsLogger Gitlab::Metrics.counter( :failed_repository_cleanup_total, 'Number of failed repository cleanup events' @@ -1059,12 +1075,16 @@ module Gitlab return unless commit_object && commit_object.type == :COMMIT + urls = gitaly_submodule_urls_for(ref) + urls && urls[path] + end + + def gitaly_submodule_urls_for(ref) gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) return unless gitmodules - found_module = GitmodulesParser.new(gitmodules.data).parse[path] - - found_module && found_module['url'] + submodules = GitmodulesParser.new(gitmodules.data).parse + submodules.transform_values { |submodule| submodule['url'] } end # Returns true if the given ref name exists diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb index 86c9f33d82a..5c73c0c66a9 100644 --- a/lib/gitlab/git/rugged_impl/blob.rb +++ b/lib/gitlab/git/rugged_impl/blob.rb @@ -16,7 +16,7 @@ module Gitlab override :tree_entry def tree_entry(repository, sha, path, limit) if use_rugged?(repository, :rugged_tree_entry) - rugged_tree_entry(repository, sha, path, limit) + execute_rugged_call(:rugged_tree_entry, repository, sha, path, limit) else super end diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb index 971a33b2e99..0eff35ab1c4 100644 --- a/lib/gitlab/git/rugged_impl/commit.rb +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -36,7 +36,7 @@ module Gitlab override :find_commit def find_commit(repo, commit_id) if use_rugged?(repo, :rugged_find_commit) - rugged_find(repo, commit_id) + execute_rugged_call(:rugged_find, repo, commit_id) else super end @@ -45,7 +45,7 @@ module Gitlab override :batch_by_oid def batch_by_oid(repo, oids) if use_rugged?(repo, :rugged_list_commits_by_oid) - rugged_batch_by_oid(repo, oids) + execute_rugged_call(:rugged_batch_by_oid, repo, oids) else super end @@ -68,7 +68,7 @@ module Gitlab override :commit_tree_entry def commit_tree_entry(path) if use_rugged?(@repository, :rugged_commit_tree_entry) - rugged_tree_entry(path) + execute_rugged_call(:rugged_tree_entry, path) else super end diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb index 9268abdfed9..8fde93e71e2 100644 --- a/lib/gitlab/git/rugged_impl/repository.rb +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -48,7 +48,7 @@ module Gitlab override :ancestor? def ancestor?(from, to) if use_rugged?(self, :rugged_commit_is_ancestor) - rugged_is_ancestor?(from, to) + execute_rugged_call(:rugged_is_ancestor?, from, to) else super end diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index f3721a3f1b7..389c9d32ccb 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -16,7 +16,7 @@ module Gitlab override :tree_entries def tree_entries(repository, sha, path, recursive) if use_rugged?(repository, :rugged_tree_entries) - tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive) + execute_rugged_call(:tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive) else super end diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb index 99091b03cd1..80b75689334 100644 --- a/lib/gitlab/git/rugged_impl/use_rugged.rb +++ b/lib/gitlab/git/rugged_impl/use_rugged.rb @@ -10,6 +10,29 @@ module Gitlab Gitlab::GitalyClient.can_use_disk?(repo.storage) end + + def execute_rugged_call(method_name, *args) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + start = Gitlab::Metrics::System.monotonic_time + + result = send(method_name, *args) # rubocop:disable GitlabSecurity/PublicSend + + duration = Gitlab::Metrics::System.monotonic_time - start + + if Gitlab::RuggedInstrumentation.active? + Gitlab::RuggedInstrumentation.increment_query_count + Gitlab::RuggedInstrumentation.query_time += duration + + Gitlab::RuggedInstrumentation.add_call_details( + feature: method_name, + args: args, + duration: duration, + backtrace: Gitlab::Profiler.clean_backtrace(caller)) + end + + result + end + end end end end diff --git a/lib/gitlab/git_logger.rb b/lib/gitlab/git_logger.rb index dac4ddd320f..545451f0dc9 100644 --- a/lib/gitlab/git_logger.rb +++ b/lib/gitlab/git_logger.rb @@ -1,13 +1,9 @@ # frozen_string_literal: true module Gitlab - class GitLogger < Gitlab::Logger + class GitLogger < JsonLogger def self.file_name_noext - 'githost' - end - - def format_message(severity, timestamp, progname, msg) - "#{timestamp.to_s(:long)} -> #{severity} -> #{msg}\n" + 'git_json' end end end diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index d98b85fecc4..2a8bcd015a8 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -27,6 +27,29 @@ module Gitlab end end + def includes_branches? + enum_for(:changes_refs).any? do |_oldrev, _newrev, ref| + Gitlab::Git.branch_ref?(ref) + end + end + + def includes_tags? + enum_for(:changes_refs).any? do |_oldrev, _newrev, ref| + Gitlab::Git.tag_ref?(ref) + end + end + + def includes_default_branch? + # If the branch doesn't have a default branch yet, we presume the + # first branch pushed will be the default. + return true unless project.default_branch.present? + + enum_for(:changes_refs).any? do |_oldrev, _newrev, ref| + Gitlab::Git.branch_ref?(ref) && + Gitlab::Git.branch_name(ref) == project.default_branch + end + end + private def deserialize_changes(changes) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index cc9503fb6de..201db9fec26 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -67,7 +67,7 @@ module Gitlab File.read(cert_file).scan(PEM_REGEX).map do |cert| OpenSSL::X509::Certificate.new(cert).to_pem rescue OpenSSL::OpenSSLError => e - Rails.logger.error "Could not load certificate #{cert_file} #{e}" + Rails.logger.error "Could not load certificate #{cert_file} #{e}" # rubocop:disable Gitlab/RailsLogger Gitlab::Sentry.track_exception(e, extra: { cert_file: cert_file }) nil end.compact @@ -211,8 +211,7 @@ module Gitlab metadata['call_site'] = feature.to_s if feature metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id - metadata['gitaly-session-id'] = session_id if Feature::Gitaly.enabled?(Feature::Gitaly::CATFILE_CACHE) - + metadata['gitaly-session-id'] = session_id metadata.merge!(Feature::Gitaly.server_feature_flags) result = { metadata: metadata } @@ -241,7 +240,7 @@ module Gitlab # Ensures that Gitaly is not being abuse through n+1 misuse etc def self.enforce_gitaly_request_limits(call_site) - # Only count limits in request-response environments (not sidekiq for example) + # Only count limits in request-response environments return unless Gitlab::SafeRequestStore.active? # This is this actual number of times this call was made. Used for information purposes only @@ -388,34 +387,34 @@ module Gitlab end def self.can_use_disk?(storage) - false - # cached_value = MUTEX.synchronize do - # @can_use_disk ||= {} - # @can_use_disk[storage] - # end + cached_value = MUTEX.synchronize do + @can_use_disk ||= {} + @can_use_disk[storage] + end - # return cached_value unless cached_value.nil? + return cached_value unless cached_value.nil? - # gitaly_filesystem_id = filesystem_id(storage) - # direct_filesystem_id = filesystem_id_from_disk(storage) + gitaly_filesystem_id = filesystem_id(storage) + direct_filesystem_id = filesystem_id_from_disk(storage) - # MUTEX.synchronize do - # @can_use_disk[storage] = gitaly_filesystem_id.present? && - # gitaly_filesystem_id == direct_filesystem_id - # end + MUTEX.synchronize do + @can_use_disk[storage] = gitaly_filesystem_id.present? && + gitaly_filesystem_id == direct_filesystem_id + end end def self.filesystem_id(storage) response = Gitlab::GitalyClient::ServerService.new(storage).info storage_status = response.storage_statuses.find { |status| status.storage_name == storage } - storage_status.filesystem_id + + storage_status&.filesystem_id end def self.filesystem_id_from_disk(storage) metadata_file = File.read(storage_metadata_file_path(storage)) metadata_hash = JSON.parse(metadata_file) metadata_hash['gitaly_filesystem_id'] - rescue Errno::ENOENT, JSON::ParserError + rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError nil end diff --git a/lib/gitlab/gitaly_client/notification_service.rb b/lib/gitlab/gitaly_client/notification_service.rb deleted file mode 100644 index 873c3e4086d..00000000000 --- a/lib/gitlab/gitaly_client/notification_service.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GitalyClient - class NotificationService - # 'repository' is a Gitlab::Git::Repository - def initialize(repository) - @gitaly_repo = repository.gitaly_repository - @storage = repository.storage - end - - def post_receive - GitalyClient.call( - @storage, - :notification_service, - :post_receive, - Gitaly::PostReceiveRequest.new(repository: @gitaly_repo) - ) - end - end - end -end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 783c2ff0915..33ca428a942 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -325,11 +325,11 @@ module Gitlab # rubocop:disable Metrics/ParameterLists def user_commit_files( user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository, force = false) + start_branch_name, start_repository, force = false, start_sha = nil) req_enum = Enumerator.new do |y| header = user_commit_files_request_header(user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository, force) + start_branch_name, start_repository, force, start_sha) y.yield Gitaly::UserCommitFilesRequest.new(header: header) @@ -445,7 +445,7 @@ module Gitlab # rubocop:disable Metrics/ParameterLists def user_commit_files_request_header( user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository, force) + start_branch_name, start_repository, force, start_sha) Gitaly::UserCommitFilesRequestHeader.new( repository: @gitaly_repo, @@ -456,7 +456,8 @@ module Gitlab commit_author_email: encode_binary(author_email), start_branch_name: encode_binary(start_branch_name), start_repository: start_repository.gitaly_repository, - force: force + force: force, + start_sha: encode_binary(start_sha) ) end # rubocop:enable Metrics/ParameterLists diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index d8e9dccb644..ca3e5b51ecc 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -5,7 +5,7 @@ module Gitlab class RepositoryService include Gitlab::EncodingHelper - MAX_MSG_SIZE = 128.kilobytes.freeze + MAX_MSG_SIZE = 128.kilobytes def initialize(repository) @repository = repository diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index a61beafae0d..826b35d685c 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -40,7 +40,7 @@ module Gitlab # otherwise hitting the rate limit will result in a thread # being blocked in a `sleep()` call for up to an hour. def initialize(token, per_page: 100, parallel: true) - @octokit = Octokit::Client.new( + @octokit = ::Octokit::Client.new( access_token: token, per_page: per_page, api_endpoint: api_endpoint @@ -139,7 +139,7 @@ module Gitlab begin yield - rescue Octokit::TooManyRequests + rescue ::Octokit::TooManyRequests raise_or_wait_for_rate_limit # This retry will only happen when running in sequential mode as we'll diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb index 6046e30d4ef..30763492235 100644 --- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -29,7 +29,7 @@ module Gitlab yield object end rescue StandardError => e - Rails.logger.error("The Lfs import process failed. #{e.message}") + Rails.logger.error("The Lfs import process failed. #{e.message}") # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index a52866c4b08..929fceaacf2 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -40,7 +40,7 @@ module Gitlab pname = project.path_with_namespace - Rails.logger + Rails.logger # rubocop:disable Gitlab/RailsLogger .info("GitHub importer finished updating repository for #{pname}") repository_updates_counter.increment diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 0e7c9ee0d00..9d925581441 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -36,6 +36,7 @@ module Gitlab description: description_for(release), created_at: release.created_at, updated_at: release.updated_at, + released_at: release.published_at, project_id: project.id } end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 41ec8741eb1..92917028851 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -38,11 +38,6 @@ module Gitlab gon.current_user_fullname = current_user.name gon.current_user_avatar_url = current_user.avatar_url end - - # Flag controls a GFM feature used across many routes. - # Pushing the flag from one place simplifies control - # and facilitates easy removal. - push_frontend_feature_flag(:gfm_embedded_metrics) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/grape_logging/loggers/client_env_logger.rb b/lib/gitlab/grape_logging/loggers/client_env_logger.rb new file mode 100644 index 00000000000..3acc6f6a2ef --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/client_env_logger.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# This is a fork of +# https://github.com/aserafin/grape_logging/blob/master/lib/grape_logging/loggers/client_env.rb +# to use remote_ip instead of ip. +module Gitlab + module GrapeLogging + module Loggers + class ClientEnvLogger < ::GrapeLogging::Loggers::Base + def parameters(request, _) + { remote_ip: request.env["HTTP_X_FORWARDED_FOR"] || request.env["REMOTE_ADDR"], ua: request.env["HTTP_USER_AGENT"] } + end + end + end + end +end diff --git a/lib/gitlab/grape_logging/loggers/perf_logger.rb b/lib/gitlab/grape_logging/loggers/perf_logger.rb index 18ea3a8d2f3..7e86b35a215 100644 --- a/lib/gitlab/grape_logging/loggers/perf_logger.rb +++ b/lib/gitlab/grape_logging/loggers/perf_logger.rb @@ -6,11 +6,30 @@ module Gitlab module Loggers class PerfLogger < ::GrapeLogging::Loggers::Base def parameters(_, _) + gitaly_data.merge(rugged_data) + end + + def gitaly_data + gitaly_calls = Gitlab::GitalyClient.get_request_count + + return {} if gitaly_calls.zero? + { gitaly_calls: Gitlab::GitalyClient.get_request_count, gitaly_duration: Gitlab::GitalyClient.query_time_ms } end + + def rugged_data + rugged_calls = Gitlab::RuggedInstrumentation.query_count + + return {} if rugged_calls.zero? + + { + rugged_calls: rugged_calls, + rugged_duration_ms: Gitlab::RuggedInstrumentation.query_time_ms + } + end end end end diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb new file mode 100644 index 00000000000..ac2a78c0f28 --- /dev/null +++ b/lib/gitlab/graphql/docs/helper.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +module Gitlab + module Graphql + module Docs + # Helper with functions to be used by HAML templates + # This includes graphql-docs gem helpers class. + # You can check the included module on: https://github.com/gjtorikian/graphql-docs/blob/v1.6.0/lib/graphql-docs/helpers.rb + module Helper + include GraphQLDocs::Helpers + + def auto_generated_comment + <<-MD.strip_heredoc + <!--- + This documentation is auto generated by a script. + + Please do not edit this file directly, check compile_docs task on lib/tasks/gitlab/graphql.rake. + ---> + MD + end + + # Some fields types are arrays of other types and are displayed + # on docs wrapped in square brackets, for example: [String!]. + # This makes GitLab docs renderer thinks they are links so here + # we change them to be rendered as: String! => Array. + def render_field_type(type) + array_type = type[/\[(.+)\]/, 1] + + if array_type + "#{array_type} => Array" + else + type + end + end + + # We are ignoring connections and built in types for now, + # they should be added when queries are generated. + def objects + graphql_object_types.select do |object_type| + !object_type[:name]["Connection"] && + !object_type[:name]["Edge"] && + !object_type[:name]["__"] + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb new file mode 100644 index 00000000000..f47a372aa19 --- /dev/null +++ b/lib/gitlab/graphql/docs/renderer.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +module Gitlab + module Graphql + module Docs + # Gitlab renderer for graphql-docs. + # Uses HAML templates to parse markdown and generate .md files. + # It uses graphql-docs helpers and schema parser, more information in https://github.com/gjtorikian/graphql-docs. + # + # Arguments: + # schema - the GraphQL schema defition. For GitLab should be: GitlabSchema.graphql_definition + # output_dir: The folder where the markdown files will be saved + # template: The path of the haml template to be parsed + class Renderer + include Gitlab::Graphql::Docs::Helper + + def initialize(schema, output_dir:, template:) + @output_dir = output_dir + @template = template + @layout = Haml::Engine.new(File.read(template)) + @parsed_schema = GraphQLDocs::Parser.new(schema, {}).parse + end + + def render + contents = @layout.render(self) + + write_file(contents) + end + + private + + def write_file(contents) + filename = File.join(@output_dir, 'index.md') + + FileUtils.mkdir_p(@output_dir) + File.write(filename, contents) + end + end + end + end +end diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml new file mode 100644 index 00000000000..cc22d43ab4f --- /dev/null +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -0,0 +1,25 @@ +-# haml-lint:disable UnnecessaryStringOutput + += auto_generated_comment + +:plain + # GraphQL API Resources + + This documentation is self-generated based on GitLab current GraphQL schema. + + The API can be explored interactively using the [GraphiQL IDE](../index.md#graphiql). + + ## Objects +\ +- objects.each do |type| + - unless type[:fields].empty? + = "### #{type[:name]}" + \ + ~ "| Name | Type | Description |" + ~ "| --- | ---- | ---------- |" + - type[:fields].each do |field| + = "| `#{field[:name]}` | #{render_field_type(field[:type][:info])} | #{field[:description]} |" + \ + + + diff --git a/lib/gitlab/graphql/loaders/batch_root_storage_statistics_loader.rb b/lib/gitlab/graphql/loaders/batch_root_storage_statistics_loader.rb new file mode 100644 index 00000000000..a0312366d66 --- /dev/null +++ b/lib/gitlab/graphql/loaders/batch_root_storage_statistics_loader.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Loaders + class BatchRootStorageStatisticsLoader + attr_reader :namespace_id + + def initialize(namespace_id) + @namespace_id = namespace_id + end + + def find + BatchLoader.for(namespace_id).batch do |namespace_ids, loader| + Namespace::RootStorageStatistics.for_namespace_ids(namespace_ids).each do |statistics| + loader.call(statistics.namespace_id, statistics) + end + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb index ab03c40c22d..941a4f434a1 100644 --- a/lib/gitlab/graphql/present/instrumentation.rb +++ b/lib/gitlab/graphql/present/instrumentation.rb @@ -23,7 +23,9 @@ module Gitlab end presenter = presented_in.presenter_class.new(object, **context.to_h) - wrapped = presented_type.class.new(presenter, context) + + # we have to use the new `authorized_new` method, as `new` is protected + wrapped = presented_type.class.authorized_new(presenter, context) old_resolver.call(wrapped, args, context) end diff --git a/lib/gitlab/graphql/representation/submodule_tree_entry.rb b/lib/gitlab/graphql/representation/submodule_tree_entry.rb new file mode 100644 index 00000000000..65716dff75d --- /dev/null +++ b/lib/gitlab/graphql/representation/submodule_tree_entry.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Representation + class SubmoduleTreeEntry < SimpleDelegator + class << self + def decorate(submodules, tree) + repository = tree.repository + submodule_links = Gitlab::SubmoduleLinks.new(repository) + + submodules.map do |submodule| + self.new(submodule, submodule_links.for(submodule, tree.sha)) + end + end + end + + def initialize(submodule, submodule_links) + @submodule_links = submodule_links + + super(submodule) + end + + def web_url + @submodule_links.first + end + + def tree_url + @submodule_links.last + end + end + end + end +end diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index 1f0deebea39..6a8e16f5a85 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -62,6 +62,7 @@ module Gitlab # Flag a project to be migrated to Hashed Storage # # @param [Project] project that will be migrated + # rubocop:disable Gitlab/RailsLogger def migrate(project) Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." @@ -69,10 +70,12 @@ module Gitlab rescue => err Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end + # rubocop:enable Gitlab/RailsLogger # Flag a project to be rolled-back to Legacy Storage # # @param [Project] project that will be rolled-back + # rubocop:disable Gitlab/RailsLogger def rollback(project) Rails.logger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..." @@ -80,6 +83,7 @@ module Gitlab rescue => err Rails.logger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end + # rubocop:enable Gitlab/RailsLogger # Returns whether we have any pending storage migration # diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 87a31a37e3f..14727b03ce9 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -19,8 +19,12 @@ module Gitlab ENV['ID_TO'] end + def self.using_ranges? + !range_from.nil? && !range_to.nil? + end + def self.range_single_item? - !range_from.nil? && range_from == range_to + using_ranges? && range_from == range_to end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb index 2bcd25cd3cc..ec4b97eaca4 100644 --- a/lib/gitlab/health_checks/db_check.rb +++ b/lib/gitlab/health_checks/db_check.rb @@ -18,11 +18,7 @@ module Gitlab def check catch_timeout 10.seconds do - if Gitlab::Database.postgresql? - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s - else - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s - end + ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s end end end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index 3588260d6eb..5a1e8c2a1dd 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -18,7 +18,7 @@ module Gitlab def metrics result, elapsed = with_timing(&method(:check)) - Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless successful?(result) + Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless successful?(result) # rubocop:disable Gitlab/RailsLogger [ metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), metric("#{metric_prefix}_success", successful?(result) ? 1 : 0), diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 41eab3658bc..84eb60f3a5d 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # This class is part of the Gitlab::HTTP wrapper. Depending on the value -# of the global setting allow_local_requests_from_hooks_and_services this adapter +# of the global setting allow_local_requests_from_web_hooks_and_services this adapter # will allow/block connection to internal IPs and/or urls. # # This functionality can be overridden by providing the setting the option @@ -38,7 +38,7 @@ module Gitlab end def allow_settings_local_requests? - Gitlab::CurrentSettings.allow_local_requests_from_hooks_and_services? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end end end diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb index 5b3f30d894a..aaade39dd62 100644 --- a/lib/gitlab/import/database_helpers.rb +++ b/lib/gitlab/import/database_helpers.rb @@ -6,9 +6,7 @@ module Gitlab # Inserts a raw row and returns the ID of the inserted row. # # attributes - The attributes/columns to set. - # relation - An ActiveRecord::Relation to use for finding the ID of the row - # when using MySQL. - # rubocop: disable CodeReuse/ActiveRecord + # relation - An ActiveRecord::Relation to use for finding the table name def insert_and_return_id(attributes, relation) # We use bulk_insert here so we can bypass any queries executed by # callbacks or validation rules, as doing this wouldn't scale when @@ -16,12 +14,8 @@ module Gitlab result = Gitlab::Database .bulk_insert(relation.table_name, [attributes], return_ids: true) - # MySQL doesn't support returning the IDs of a bulk insert in a way that - # is not a pain, so in this case we'll issue an extra query instead. - result.first || - relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first + result.first end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index f63a5ece71e..bb46bd657e8 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -4,7 +4,9 @@ module Gitlab module ImportExport extend self - # For every version update, the version history in import_export.md has to be kept up to date. + # For every version update the version history in these docs must be kept up to date: + # - development/import_export.md + # - user/project/settings/import_export.md VERSION = '0.2.4'.freeze FILENAME_LIMIT = 50 @@ -28,6 +30,14 @@ module Gitlab "project.bundle" end + def lfs_objects_filename + "lfs-objects.json" + end + + def lfs_objects_storage + "lfs-objects" + end + def config_file Rails.root.join('lib/gitlab/import_export/import_export.yml') end diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index c28a1674018..b2fe9592c06 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -3,7 +3,7 @@ module Gitlab module ImportExport class AttributeCleaner - ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + ['group_id'] + ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + %w[group_id commit_id] PROHIBITED_REFERENCES = Regexp.union(/\Acached_markdown_version\Z/, /_id\Z/, /_html\Z/).freeze def self.clean(*args) diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 409243e68a5..42cd94add79 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -45,7 +45,7 @@ module Gitlab end def key_from_hash(value) - value.is_a?(Hash) ? value.keys.first : value + value.is_a?(Hash) ? value.first.first : value end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 01437c67fa9..bd0f3e70749 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -37,11 +37,11 @@ project_tree: - :user - merge_requests: - :metrics - - :suggestions - notes: - :author - events: - :push_event_payload + - :suggestions - merge_request_diff: - :merge_request_diff_commits - :merge_request_diff_files @@ -80,6 +80,10 @@ project_tree: - :ci_cd_settings - :error_tracking_setting - :metrics_setting + - boards: + - lists: + - label: + - :priorities # Only include the following attributes for the models specified. included_attributes: @@ -133,6 +137,7 @@ excluded_attributes: - :packages_enabled - :mirror_last_update_at - :mirror_last_successful_update_at + - :emails_disabled namespaces: - :runners_token - :runners_token_encrypted @@ -216,6 +221,8 @@ methods: - :action project_badges: - :type + lists: + - :list_type # EE specific relationships and settings to include. All of this will be merged # into the previous structures if EE is used. diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index b145f37c052..a92e3862361 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -27,7 +27,7 @@ module Gitlab # {:merge_requests=>[:merge_request_diff, :notes]} def process_model_objects(model_object_hash) json_config_hash = {} - current_key = model_object_hash.keys.first + current_key = model_object_hash.first.first model_object_hash.values.flatten.each do |model_object| @attributes_finder.parse(current_key) { |hash| json_config_hash[current_key] ||= hash } diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index 345c7880e30..1de8a5bf9ec 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -3,6 +3,10 @@ module Gitlab module ImportExport class LfsRestorer + include Gitlab::Utils::StrongMemoize + + attr_accessor :project, :shared + def initialize(project:, shared:) @project = project @shared = shared @@ -17,7 +21,7 @@ module Gitlab true rescue => e - @shared.error(e) + shared.error(e) false end @@ -29,16 +33,57 @@ module Gitlab lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size) lfs_object.file = File.open(path) unless lfs_object.file&.exists? + lfs_object.save! if lfs_object.changed? - @project.all_lfs_objects << lfs_object + repository_types(oid).each do |repository_type| + LfsObjectsProject.create!( + project: project, + lfs_object: lfs_object, + repository_type: repository_type + ) + end + end + + def repository_types(oid) + # We allow support for imports created before the `lfs-objects.json` + # file was generated. In this case, the restorer will link an LFS object + # with a single `lfs_objects_projects` relation. + # + # This allows us backwards-compatibility without version bumping. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30830#note_192608870 + return ['project'] unless has_lfs_json? + + lfs_json[oid] end def lfs_file_paths @lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*") end + def has_lfs_json? + strong_memoize(:has_lfs_json) do + File.exist?(lfs_json_path) + end + end + + def lfs_json + return {} unless has_lfs_json? + + @lfs_json ||= + begin + json = IO.read(lfs_json_path) + ActiveSupport::JSON.decode(json) + rescue + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + end + def lfs_storage_path - File.join(@shared.export_path, 'lfs-objects') + File.join(shared.export_path, ImportExport.lfs_objects_storage) + end + + def lfs_json_path + File.join(shared.export_path, ImportExport.lfs_objects_filename) end end end diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 954f6f00078..18c590e1ca9 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -5,25 +5,40 @@ module Gitlab class LfsSaver include Gitlab::ImportExport::CommandLineUtil + attr_accessor :lfs_json, :project, :shared + + BATCH_SIZE = 100 + def initialize(project:, shared:) @project = project @shared = shared + @lfs_json = {} end def save - @project.all_lfs_objects.each do |lfs_object| - save_lfs_object(lfs_object) + project.all_lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| + batch.each do |lfs_object| + save_lfs_object(lfs_object) + end + + append_lfs_json_for_batch(batch) if write_lfs_json_enabled? end + write_lfs_json if write_lfs_json_enabled? + true rescue => e - @shared.error(e) + shared.error(e) false end private + def write_lfs_json_enabled? + ::Feature.enabled?(:export_lfs_objects_projects, default_enabled: true) + end + def save_lfs_object(lfs_object) if lfs_object.local_store? copy_file_for_lfs_object(lfs_object) @@ -45,12 +60,36 @@ module Gitlab copy_files(lfs_object.file.path, destination_path_for_object(lfs_object)) end + def append_lfs_json_for_batch(lfs_objects_batch) + lfs_objects_projects = LfsObjectsProject + .select('lfs_objects.oid, array_agg(distinct lfs_objects_projects.repository_type) as repository_types') + .joins(:lfs_object) + .where(project: project, lfs_object: lfs_objects_batch) + .group('lfs_objects.oid') + + lfs_objects_projects.each do |group| + oid = group.oid + + lfs_json[oid] ||= [] + lfs_json[oid] += group.repository_types + end + end + + def write_lfs_json + mkdir_p(shared.export_path) + File.write(lfs_json_path, lfs_json.to_json) + end + def destination_path_for_object(lfs_object) File.join(lfs_export_path, lfs_object.oid) end def lfs_export_path - File.join(@shared.export_path, 'lfs-objects') + File.join(shared.export_path, ImportExport.lfs_objects_storage) + end + + def lfs_json_path + File.join(shared.export_path, ImportExport.lfs_objects_filename) end end end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index a154de5419e..4e976cfca3a 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -35,7 +35,7 @@ module Gitlab end def include?(old_author_id) - map.keys.include?(old_author_id) && map[old_author_id] != default_user_id + map.has_key?(old_author_id) && map[old_author_id] != default_user_id end private @@ -50,6 +50,8 @@ module Gitlab @project.project_members.destroy_all # rubocop: disable DestroyAll ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true) + rescue => e + raise e, "Error adding importer user to project members. #{e.message}" end def add_team_member(member, existing_user = nil) diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index deb2f59f05f..0b534a5bafc 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -43,7 +43,7 @@ module Gitlab target_ref = Gitlab::Git::BRANCH_REF_PREFIX + @merge_request.source_branch unless @project.repository.fetch_source_branch!(@project.repository, @diff_head_sha, target_ref) - Rails.logger.warn("Import/Export warning: Failed to create #{target_ref} for MR: #{@merge_request.iid}") + Rails.logger.warn("Import/Export warning: Failed to create #{target_ref} for MR: #{@merge_request.iid}") # rubocop:disable Gitlab/RailsLogger end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 20caadb89c0..91fe4e5d074 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -20,7 +20,7 @@ module Gitlab json = IO.read(@path) @tree_hash = ActiveSupport::JSON.decode(json) rescue => e - Rails.logger.error("Import/Export error: #{e.message}") + Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger raise Gitlab::ImportExport::Error.new('Incorrect JSON format') end @@ -130,6 +130,7 @@ module Gitlab def visibility_level level = override_params['visibility_level'] || json_params['visibility_level'] || @project.visibility_level level = @project.group.visibility_level if @project.group && level.to_i > @project.group.visibility_level + level = Gitlab::VisibilityLevel::PRIVATE if level == Gitlab::VisibilityLevel::INTERNAL && Gitlab::CurrentSettings.restricted_visibility_levels.include?(level) { 'visibility_level' => level } end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index efd3f550a22..0be49e27acb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -28,7 +28,7 @@ module Gitlab links: 'Releases::Link', metrics_setting: 'ProjectMetricsSetting' }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id owner_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze @@ -78,6 +78,9 @@ module Gitlab def create return if unknown_service? + # Do not import legacy triggers + return if !Feature.enabled?(:use_legacy_pipeline_triggers, @project) && legacy_trigger? + setup_models generate_imported_object @@ -182,7 +185,7 @@ module Gitlab return unless EXISTING_OBJECT_CHECK.include?(@relation_name) return unless @relation_hash['group_id'] - @relation_hash['group_id'] = @project.group&.id + @relation_hash['group_id'] = @project.namespace_id end def reset_tokens! @@ -278,6 +281,10 @@ module Gitlab !Object.const_defined?(parsed_relation_hash['type']) end + def legacy_trigger? + @relation_name == 'Ci::Trigger' && @relation_hash['owner_id'].nil? + end + def find_or_create_object! return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 72f575db095..bea7a7cce65 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -18,7 +18,7 @@ module Gitlab if compress_and_save remove_export_path - Rails.logger.info("Saved project export #{archive_file}") + Rails.logger.info("Saved project export #{archive_file}") # rubocop:disable Gitlab/RailsLogger save_upload else diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 6d978d00ea5..86ea7a30e69 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -36,7 +36,7 @@ module Gitlab def different_version?(version) Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) rescue => e - Rails.logger.error("Import/Export error: #{e.message}") + Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger raise Gitlab::ImportExport::Error.new('Incorrect VERSION format') end end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb new file mode 100644 index 00000000000..e6a5facb2a5 --- /dev/null +++ b/lib/gitlab/instrumentation_helper.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module InstrumentationHelper + extend self + + KEYS = %i(gitaly_calls gitaly_duration rugged_calls rugged_duration_ms).freeze + + def add_instrumentation_data(payload) + gitaly_calls = Gitlab::GitalyClient.get_request_count + + if gitaly_calls > 0 + payload[:gitaly_calls] = gitaly_calls + payload[:gitaly_duration] = Gitlab::GitalyClient.query_time_ms + end + + rugged_calls = Gitlab::RuggedInstrumentation.query_count + + if rugged_calls > 0 + payload[:rugged_calls] = rugged_calls + payload[:rugged_duration_ms] = Gitlab::RuggedInstrumentation.query_time_ms + end + end + end +end diff --git a/lib/gitlab/kubernetes/default_namespace.rb b/lib/gitlab/kubernetes/default_namespace.rb new file mode 100644 index 00000000000..c95362b024b --- /dev/null +++ b/lib/gitlab/kubernetes/default_namespace.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class DefaultNamespace + attr_reader :cluster, :project + + delegate :platform_kubernetes, to: :cluster + + ## + # Ideally we would just use an environment record here instead of + # passing a project and name/slug separately, but we need to be able + # to look up namespaces before the environment has been persisted. + def initialize(cluster, project:) + @cluster = cluster + @project = project + end + + def from_environment_name(name) + from_environment_slug(generate_slug(name)) + end + + def from_environment_slug(slug) + default_platform_namespace(slug) || default_project_namespace(slug) + end + + private + + def default_platform_namespace(slug) + return unless platform_kubernetes&.namespace.present? + + if cluster.managed? && cluster.namespace_per_environment? + "#{platform_kubernetes.namespace}-#{slug}" + else + platform_kubernetes.namespace + end + end + + def default_project_namespace(slug) + namespace_slug = "#{project.path}-#{project.id}".downcase + + if cluster.namespace_per_environment? + namespace_slug += "-#{slug}" + end + + Gitlab::NamespaceSanitizer.sanitize(namespace_slug) + end + + ## + # Environment slug can be predicted given an environment + # name, so even if the environment isn't persisted yet we + # still know what to look for. + def generate_slug(name) + Gitlab::Slug::Environment.new(name).generate + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 42c4745ff98..6e4286589c1 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -3,8 +3,8 @@ module Gitlab module Kubernetes module Helm - HELM_VERSION = '2.12.3'.freeze - KUBECTL_VERSION = '1.11.7'.freeze + HELM_VERSION = '2.14.3'.freeze + KUBECTL_VERSION = '1.11.10'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze SERVICE_ACCOUNT = 'tiller'.freeze CLUSTER_ROLE_BINDING = 'tiller-admin'.freeze diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb index 9940272a8bf..6ae68306a9b 100644 --- a/lib/gitlab/kubernetes/helm/client_command.rb +++ b/lib/gitlab/kubernetes/helm/client_command.rb @@ -13,15 +13,27 @@ module Gitlab end def wait_for_tiller_command + helm_check = ['helm', 'version', *optional_tls_flags].shelljoin # This is necessary to give Tiller time to restart after upgrade. # Ideally we'd be able to use --wait but cannot because of # https://github.com/helm/helm/issues/4855 - 'for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done' + "for i in $(seq 1 30); do #{helm_check} && break; sleep 1s; echo \"Retrying ($i)...\"; done" end def repository_command ['helm', 'repo', 'add', name, repository].shelljoin if repository end + + def optional_tls_flags + return [] unless files.key?(:'ca.pem') + + [ + '--tls', + '--tls-ca-cert', "#{files_dir}/ca.pem", + '--tls-cert', "#{files_dir}/cert.pem", + '--tls-key', "#{files_dir}/key.pem" + ] + end end end end diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb index 876994d2678..dcf22e7abb6 100644 --- a/lib/gitlab/kubernetes/helm/delete_command.rb +++ b/lib/gitlab/kubernetes/helm/delete_command.rb @@ -7,19 +7,24 @@ module Gitlab include BaseCommand include ClientCommand + attr_reader :predelete, :postdelete attr_accessor :name, :files - def initialize(name:, rbac:, files:) + def initialize(name:, rbac:, files:, predelete: nil, postdelete: nil) @name = name @files = files @rbac = rbac + @predelete = predelete + @postdelete = postdelete end def generate_script super + [ init_command, wait_for_tiller_command, - delete_command + predelete, + delete_command, + postdelete ].compact.join("\n") end @@ -38,17 +43,6 @@ module Gitlab command.shelljoin end - - def optional_tls_flags - return [] unless files.key?(:'ca.pem') - - [ - '--tls', - '--tls-ca-cert', "#{files_dir}/ca.pem", - '--tls-cert', "#{files_dir}/cert.pem", - '--tls-key', "#{files_dir}/key.pem" - ] - end end end end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index e33ba9305ce..f572bc43533 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -27,9 +27,9 @@ module Gitlab wait_for_tiller_command, repository_command, repository_update_command, - preinstall_command, + preinstall, install_command, - postinstall_command + postinstall ].compact.join("\n") end @@ -58,14 +58,6 @@ module Gitlab command.shelljoin end - def preinstall_command - preinstall.join("\n") if preinstall - end - - def postinstall_command - postinstall.join("\n") if postinstall - end - def install_flag ['--install'] end @@ -95,17 +87,6 @@ module Gitlab ['--version', version] end - - def optional_tls_flags - return [] unless files.key?(:'ca.pem') - - [ - '--tls', - '--tls-ca-cert', "#{files_dir}/ca.pem", - '--tls-cert', "#{files_dir}/cert.pem", - '--tls-key', "#{files_dir}/key.pem" - ] - end end end end diff --git a/lib/gitlab/kubernetes/helm/reset_command.rb b/lib/gitlab/kubernetes/helm/reset_command.rb new file mode 100644 index 00000000000..a35ffa34c58 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/reset_command.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + class ResetCommand + include BaseCommand + include ClientCommand + + attr_reader :name, :files + + def initialize(name:, rbac:, files:) + @name = name + @files = files + @rbac = rbac + end + + def generate_script + super + [ + reset_helm_command, + delete_tiller_replicaset + ].join("\n") + end + + def rbac? + @rbac + end + + def pod_name + "uninstall-#{name}" + end + + private + + # This method can be delete once we upgrade Helm to > 12.13.0 + # https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/27096#note_159695900 + # + # Tracking this method to be removed here: + # https://gitlab.com/gitlab-org/gitlab-ce/issues/52791#note_199374155 + def delete_tiller_replicaset + delete_args = %w[replicaset -n gitlab-managed-apps -l name=tiller] + + Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) + end + + def reset_helm_command + command = %w[helm reset] + optional_tls_flags + + command.shelljoin + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index de14df56555..64317225ec6 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -59,6 +59,13 @@ module Gitlab # RBAC methods delegates to the apis/rbac.authorization.k8s.io api # group client + delegate :create_role, + :get_role, + :update_role, + to: :rbac_client + + # RBAC methods delegates to the apis/rbac.authorization.k8s.io api + # group client delegate :create_role_binding, :get_role_binding, :update_role_binding, @@ -121,7 +128,7 @@ module Gitlab private def validate_url! - return if Gitlab::CurrentSettings.allow_local_requests_from_hooks_and_services? + return if Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? Gitlab::UrlBlocker.validate!(api_prefix, allow_local_network: false) end diff --git a/lib/gitlab/kubernetes/kubectl_cmd.rb b/lib/gitlab/kubernetes/kubectl_cmd.rb new file mode 100644 index 00000000000..981eb5681dc --- /dev/null +++ b/lib/gitlab/kubernetes/kubectl_cmd.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module KubectlCmd + class << self + def delete(*args) + %w(kubectl delete).concat(args).shelljoin + end + + def apply_file(filename, *args) + raise ArgumentError, "filename is not present" unless filename.present? + + %w(kubectl apply -f).concat([filename], args).shelljoin + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/role.rb b/lib/gitlab/kubernetes/role.rb new file mode 100644 index 00000000000..096f60f0372 --- /dev/null +++ b/lib/gitlab/kubernetes/role.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class Role + def initialize(name:, namespace:, rules:) + @name = name + @namespace = namespace + @rules = rules + end + + def generate + ::Kubeclient::Resource.new( + metadata: { name: name, namespace: namespace }, + rules: rules + ) + end + + private + + attr_reader :name, :namespace, :rules + end + end +end diff --git a/lib/gitlab/kubernetes/role_binding.rb b/lib/gitlab/kubernetes/role_binding.rb index cb0cb42d007..0404fb4453c 100644 --- a/lib/gitlab/kubernetes/role_binding.rb +++ b/lib/gitlab/kubernetes/role_binding.rb @@ -3,9 +3,10 @@ module Gitlab module Kubernetes class RoleBinding - def initialize(name:, role_name:, namespace:, service_account_name:) + def initialize(name:, role_name:, role_kind:, namespace:, service_account_name:) @name = name @role_name = role_name + @role_kind = role_kind @namespace = namespace @service_account_name = service_account_name end @@ -20,7 +21,7 @@ module Gitlab private - attr_reader :name, :role_name, :namespace, :service_account_name + attr_reader :name, :role_name, :role_kind, :namespace, :service_account_name def metadata { name: name, namespace: namespace } @@ -29,7 +30,7 @@ module Gitlab def role_ref { apiGroup: 'rbac.authorization.k8s.io', - kind: 'ClusterRole', + kind: role_kind, name: role_name } end diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index bbdd094e33b..b23efd64dee 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -101,7 +101,7 @@ module Gitlab # GitHub Rate Limit API returns 404 when the rate limit is # disabled. In this case we just want to return gracefully # instead of spitting out an error. - rescue Octokit::NotFound + rescue ::Octokit::NotFound nil end diff --git a/lib/gitlab/lets_encrypt.rb b/lib/gitlab/lets_encrypt.rb index cdf24f24647..08ad2ab91b0 100644 --- a/lib/gitlab/lets_encrypt.rb +++ b/lib/gitlab/lets_encrypt.rb @@ -2,15 +2,8 @@ module Gitlab module LetsEncrypt - def self.enabled?(pages_domain = nil) - return false unless Gitlab::CurrentSettings.lets_encrypt_terms_of_service_accepted - - return false unless Feature.enabled?(:pages_auto_ssl) - - # If no domain is passed, just check whether we're enabled globally - return true unless pages_domain - - !!pages_domain.project && Feature.enabled?(:pages_auto_ssl_for_project, pages_domain.project) + def self.enabled? + Gitlab::CurrentSettings.lets_encrypt_terms_of_service_accepted end end end diff --git a/lib/gitlab/markdown_cache/active_record/extension.rb b/lib/gitlab/markdown_cache/active_record/extension.rb index f3abe631779..233d3bf1ac7 100644 --- a/lib/gitlab/markdown_cache/active_record/extension.rb +++ b/lib/gitlab/markdown_cache/active_record/extension.rb @@ -26,10 +26,6 @@ module Gitlab attrs end - def changed_markdown_fields - changed_attributes.keys.map(&:to_s) & cached_markdown_fields.markdown_fields.map(&:to_s) - end - def write_markdown_field(field_name, value) write_attribute(field_name, value) end diff --git a/lib/gitlab/markdown_cache/redis/extension.rb b/lib/gitlab/markdown_cache/redis/extension.rb index 97fc23343b4..af3237f4ba6 100644 --- a/lib/gitlab/markdown_cache/redis/extension.rb +++ b/lib/gitlab/markdown_cache/redis/extension.rb @@ -36,8 +36,8 @@ module Gitlab false end - def changed_markdown_fields - [] + def changed_attributes + {} end def cached_markdown diff --git a/lib/gitlab/metrics/dashboard/base_service.rb b/lib/gitlab/metrics/dashboard/base_service.rb deleted file mode 100644 index 0628e82e592..00000000000 --- a/lib/gitlab/metrics/dashboard/base_service.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -# Searches a projects repository for a metrics dashboard and formats the output. -# Expects any custom dashboards will be located in `.gitlab/dashboards` -module Gitlab - module Metrics - module Dashboard - class BaseService < ::BaseService - PROCESSING_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardProcessingError - NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError - - def get_dashboard - return error('Insufficient permissions.', :unauthorized) unless allowed? - - success(dashboard: process_dashboard) - rescue NOT_FOUND_ERROR - error("#{dashboard_path} could not be found.", :not_found) - rescue PROCESSING_ERROR => e - error(e.message, :unprocessable_entity) - end - - # Summary of all known dashboards for the service. - # @return [Array<Hash>] ex) [{ path: String, default: Boolean }] - def self.all_dashboard_paths(_project) - raise NotImplementedError - end - - # Returns an un-processed dashboard from the cache. - def raw_dashboard - Gitlab::Metrics::Dashboard::Cache.fetch(cache_key) { get_raw_dashboard } - end - - private - - # Determines whether users should be able to view - # dashboards at all. - def allowed? - Ability.allowed?(current_user, :read_environment, project) - end - - # Returns a new dashboard Hash, supplemented with DB info - def process_dashboard - Gitlab::Metrics::Dashboard::Processor - .new(project, params[:environment], raw_dashboard) - .process(insert_project_metrics: insert_project_metrics?) - end - - # @return [String] Relative filepath of the dashboard yml - def dashboard_path - params[:dashboard_path] - end - - # @return [Hash] an unmodified dashboard - def get_raw_dashboard - raise NotImplementedError - end - - # @return [String] - def cache_key - raise NotImplementedError - end - - # Determines whether custom metrics should be included - # in the processed output. - # @return [Boolean] - def insert_project_metrics? - false - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/defaults.rb b/lib/gitlab/metrics/dashboard/defaults.rb new file mode 100644 index 00000000000..3c39a7c6911 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/defaults.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# Central point for managing default attributes from within +# the metrics dashboard module. +module Gitlab + module Metrics + module Dashboard + module Defaults + DEFAULT_PANEL_TYPE = 'area-chart' + DEFAULT_PANEL_WEIGHT = 0 + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/dynamic_dashboard_service.rb b/lib/gitlab/metrics/dashboard/dynamic_dashboard_service.rb deleted file mode 100644 index 81ed8922e17..00000000000 --- a/lib/gitlab/metrics/dashboard/dynamic_dashboard_service.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -# Responsible for returning a filtered system dashboard -# containing only the default embedded metrics. In future, -# this class may be updated to support filtering to -# alternate metrics/panels. -# -# Why isn't this filtering in a processing stage? By filtering -# here, we ensure the dynamically-determined dashboard is cached. -# -# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. -module Gitlab - module Metrics - module Dashboard - class DynamicDashboardService < Gitlab::Metrics::Dashboard::BaseService - # For the default filtering for embedded metrics, - # uses the 'id' key in dashboard-yml definition for - # identification. - DEFAULT_EMBEDDED_METRICS_IDENTIFIERS = %w( - system_metrics_kubernetes_container_memory_total - system_metrics_kubernetes_container_cores_total - ).freeze - - # Returns a new dashboard with only the matching - # metrics from the system dashboard, stripped of groups. - # @return [Hash] - def raw_dashboard - panels = panel_groups.each_with_object([]) do |group, panels| - matched_panels = group['panels'].select { |panel| matching_panel?(panel) } - - panels.concat(matched_panels) - end - - { 'panel_groups' => [{ 'panels' => panels }] } - end - - def cache_key - "dynamic_metrics_dashboard_#{metric_identifiers.join('_')}" - end - - private - - # Returns an array of the panels groups on the - # system dashboard - def panel_groups - Gitlab::Metrics::Dashboard::SystemDashboardService - .new(project, nil) - .raw_dashboard['panel_groups'] - end - - # Identifies a panel as "matching" if any metric ids in - # the panel is in the list of identifiers to collect. - def matching_panel?(panel) - panel['metrics'].any? do |metric| - metric_identifiers.include?(metric['id']) - end - end - - def metric_identifiers - DEFAULT_EMBEDDED_METRICS_IDENTIFIERS - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/errors.rb b/lib/gitlab/metrics/dashboard/errors.rb new file mode 100644 index 00000000000..1739a4e6738 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/errors.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +# Central point for managing errors from within the metrics +# dashboard module. Handles errors from dashboard retrieval +# and processing steps, as well as defines shared error classes. +module Gitlab + module Metrics + module Dashboard + module Errors + PanelNotFoundError = Class.new(StandardError) + + PROCESSING_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardProcessingError + NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError + + def handle_errors(error) + case error + when PROCESSING_ERROR + error(error.message, :unprocessable_entity) + when NOT_FOUND_ERROR + error("#{dashboard_path} could not be found.", :not_found) + when PanelNotFoundError + error(error.message, :not_found) + else + raise error + end + end + + def panels_not_found!(opts) + raise PanelNotFoundError.new("No panels matching properties #{opts}") + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index d7491d1553d..66c4d662a6c 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -12,21 +12,37 @@ module Gitlab # @param project [Project] # @param user [User] # @param environment [Environment] - # @param opts - dashboard_path [String] Path at which the - # dashboard can be found. Nil values will - # default to the system dashboard. - # @param opts - embedded [Boolean] Determines whether the + # @param options - embedded [Boolean] Determines whether the # dashboard is to be rendered as part of an # issue or location other than the primary # metrics dashboard UI. Returns only the # Memory/CPU charts of the system dash. + # @param options - dashboard_path [String] Path at which the + # dashboard can be found. Nil values will + # default to the system dashboard. + # @param options - group [String] Title of the group + # to which a panel might belong. Used by + # embedded dashboards. + # @param options - title [String] Title of the panel. + # Used by embedded dashboards. + # @param options - y_label [String] Y-Axis label of + # a panel. Used by embedded dashboards. # @return [Hash] - def find(project, user, environment, dashboard_path: nil, embedded: false) - service_for_path(dashboard_path, embedded: embedded) - .new(project, user, environment: environment, dashboard_path: dashboard_path) + def find(project, user, environment, options = {}) + service_for(options) + .new(project, user, options.merge(environment: environment)) .get_dashboard end + # Returns a dashboard without any supplemental info. + # Returns only full, yml-defined dashboards. + # @return [Hash] + def find_raw(project, dashboard_path: nil) + service_for(dashboard_path: dashboard_path) + .new(project, nil, dashboard_path: dashboard_path) + .raw_dashboard + end + # Summary of all known dashboards. # @return [Array<Hash>] ex) [{ path: String, # display_name: String, @@ -46,27 +62,16 @@ module Gitlab private - def service_for_path(dashboard_path, embedded:) - return dynamic_service if embedded - return system_service if system_dashboard?(dashboard_path) - - project_service - end - def system_service - Gitlab::Metrics::Dashboard::SystemDashboardService + ::Metrics::Dashboard::SystemDashboardService end def project_service - Gitlab::Metrics::Dashboard::ProjectDashboardService - end - - def dynamic_service - Gitlab::Metrics::Dashboard::DynamicDashboardService + ::Metrics::Dashboard::ProjectDashboardService end - def system_dashboard?(filepath) - !filepath || system_service.system_dashboard?(filepath) + def service_for(options) + Gitlab::Metrics::Dashboard::ServiceSelector.call(options) end end end diff --git a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb deleted file mode 100644 index 5a1c4ecf886..00000000000 --- a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -# Searches a projects repository for a metrics dashboard and formats the output. -# Expects any custom dashboards will be located in `.gitlab/dashboards` -# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. -module Gitlab - module Metrics - module Dashboard - class ProjectDashboardService < Gitlab::Metrics::Dashboard::BaseService - DASHBOARD_ROOT = ".gitlab/dashboards" - - class << self - def all_dashboard_paths(project) - file_finder(project) - .list_files_for(DASHBOARD_ROOT) - .map do |filepath| - { - path: filepath, - display_name: name_for_path(filepath), - default: false - } - end - end - - def file_finder(project) - Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, '.yml') - end - - # Grabs the filepath after the base directory. - def name_for_path(filepath) - filepath.delete_prefix("#{DASHBOARD_ROOT}/") - end - end - - private - - # Searches the project repo for a custom-defined dashboard. - def get_raw_dashboard - yml = self.class.file_finder(project).read(dashboard_path) - - YAML.safe_load(yml) - end - - def cache_key - "project_#{project.id}_metrics_dashboard_#{dashboard_path}" - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb new file mode 100644 index 00000000000..934ba9145a2 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/service_selector.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# Responsible for determining which dashboard service should +# be used to fetch or generate a dashboard hash. +# The services can be considered in two categories - embeds +# and dashboards. Embeds are all portions of dashboards. +module Gitlab + module Metrics + module Dashboard + class ServiceSelector + SERVICES = ::Metrics::Dashboard + + class << self + include Gitlab::Utils::StrongMemoize + + # Returns a class which inherits from the BaseService + # class that can be used to obtain a dashboard. + # @return [Gitlab::Metrics::Dashboard::Services::BaseService] + def call(params) + return SERVICES::CustomMetricEmbedService if custom_metric_embed?(params) + return SERVICES::DynamicEmbedService if dynamic_embed?(params) + return SERVICES::DefaultEmbedService if params[:embedded] + return SERVICES::SystemDashboardService if system_dashboard?(params[:dashboard_path]) + return SERVICES::ProjectDashboardService if params[:dashboard_path] + + default_service + end + + private + + def default_service + SERVICES::SystemDashboardService + end + + def system_dashboard?(filepath) + SERVICES::SystemDashboardService.system_dashboard?(filepath) + end + + def custom_metric_embed?(params) + SERVICES::CustomMetricEmbedService.valid_params?(params) + end + + def dynamic_embed?(params) + SERVICES::DynamicEmbedService.valid_params?(params) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb index 0db7b176e8d..514ed50e58d 100644 --- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -5,11 +5,11 @@ module Gitlab module Dashboard module Stages class BaseStage + include Gitlab::Metrics::Dashboard::Defaults + DashboardProcessingError = Class.new(StandardError) LayoutError = Class.new(DashboardProcessingError) - DEFAULT_PANEL_TYPE = 'area-chart' - attr_reader :project, :environment, :dashboard def initialize(project, environment, dashboard) diff --git a/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb index 221610a14d1..643be309992 100644 --- a/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb @@ -97,7 +97,7 @@ module Gitlab end def new_metric(metric) - metric.queries.first.merge(metric_id: metric.id) + metric.to_metric_hash end end end diff --git a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb deleted file mode 100644 index 82421572f4a..00000000000 --- a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -# Fetches the system metrics dashboard and formats the output. -# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. -module Gitlab - module Metrics - module Dashboard - class SystemDashboardService < Gitlab::Metrics::Dashboard::BaseService - SYSTEM_DASHBOARD_PATH = 'config/prometheus/common_metrics.yml' - SYSTEM_DASHBOARD_NAME = 'Default' - - class << self - def all_dashboard_paths(_project) - [{ - path: SYSTEM_DASHBOARD_PATH, - display_name: SYSTEM_DASHBOARD_NAME, - default: true - }] - end - - def system_dashboard?(filepath) - filepath == SYSTEM_DASHBOARD_PATH - end - end - - private - - def dashboard_path - SYSTEM_DASHBOARD_PATH - end - - # Returns the base metrics shipped with every GitLab service. - def get_raw_dashboard - yml = File.read(Rails.root.join(dashboard_path)) - - YAML.safe_load(yml) - end - - def cache_key - "metrics_dashboard_#{dashboard_path}" - end - - def insert_project_metrics? - true - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb index b197e7ca86b..94f8b2e02b1 100644 --- a/lib/gitlab/metrics/dashboard/url.rb +++ b/lib/gitlab/metrics/dashboard/url.rb @@ -21,14 +21,26 @@ module Gitlab \/(?<environment>\d+) \/metrics (?<query> - \?[a-z0-9_=-]+ - (&[a-z0-9_=-]+)* + \?[a-zA-Z0-9%.()+_=-]+ + (&[a-zA-Z0-9%.()+_=-]+)* )? (?<anchor>\#[a-z0-9_-]+)? ) }x end + # Parses query params out from full url string into hash. + # + # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group' + # --> { title: 'Title', group: 'Group' } + def parse_query(url) + query_string = URI.parse(url).query.to_s + + CGI.parse(query_string) + .transform_values { |value| value.first } + .symbolize_keys + end + # Builds a metrics dashboard url based on the passed in arguments def build_dashboard_url(*args) Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args) diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 6a062e93f0f..d7d848d2833 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -19,7 +19,7 @@ module Gitlab def safe_sample sample rescue => e - Rails.logger.warn("#{self.class}: #{e}, stopping") + Rails.logger.warn("#{self.class}: #{e}, stopping") # rubocop:disable Gitlab/RailsLogger stop end diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index 5138b37f83e..1eae0a7bf45 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -15,19 +15,14 @@ module Gitlab @last_step = nil @metrics = [] - - @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) - @last_major_gc = Delta.new(GC.stat[:major_gc_count]) end def sample sample_memory_usage sample_file_descriptors - sample_gc flush ensure - GC::Profiler.clear @metrics.clear end @@ -43,23 +38,6 @@ module Gitlab add_metric('file_descriptors', value: System.file_descriptor_count) end - def sample_gc - time = GC::Profiler.total_time * 1000.0 - stats = GC.stat.merge(total_time: time) - - # We want the difference of GC runs compared to the last sample, not the - # total amount since the process started. - stats[:minor_gc_count] = - @last_minor_gc.compared_with(stats[:minor_gc_count]) - - stats[:major_gc_count] = - @last_major_gc.compared_with(stats[:major_gc_count]) - - stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count] - - add_metric('gc_statistics', stats) - end - def add_metric(series, values, tags = {}) prefix = sidekiq? ? 'sidekiq_' : 'rails_' diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index 25e40c70230..8a24d4f3663 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -15,7 +15,6 @@ module Gitlab puma_workers: ::Gitlab::Metrics.gauge(:puma_workers, 'Total number of workers'), puma_running_workers: ::Gitlab::Metrics.gauge(:puma_running_workers, 'Number of active workers'), puma_stale_workers: ::Gitlab::Metrics.gauge(:puma_stale_workers, 'Number of stale workers'), - puma_phase: ::Gitlab::Metrics.gauge(:puma_phase, 'Phase number (increased during phased restarts)'), puma_running: ::Gitlab::Metrics.gauge(:puma_running, 'Number of running threads'), puma_queued_connections: ::Gitlab::Metrics.gauge(:puma_queued_connections, 'Number of connections in that worker\'s "todo" set waiting for a worker thread'), puma_active_connections: ::Gitlab::Metrics.gauge(:puma_active_connections, 'Number of threads processing a request'), @@ -43,7 +42,7 @@ module Gitlab def puma_stats Puma.stats rescue NoMethodError - Rails.logger.info "PumaSampler: stats are not available yet, waiting for Puma to boot" + Rails.logger.info "PumaSampler: stats are not available yet, waiting for Puma to boot" # rubocop:disable Gitlab/RailsLogger nil end @@ -54,7 +53,6 @@ module Gitlab last_status = worker['last_status'] labels = { worker: "worker_#{worker['index']}" } - metrics[:puma_phase].set(labels, worker['phase']) set_worker_metrics(last_status, labels) if last_status.present? end end @@ -76,7 +74,6 @@ module Gitlab metrics[:puma_workers].set(labels, stats['workers']) metrics[:puma_running_workers].set(labels, stats['booted_workers']) metrics[:puma_stale_workers].set(labels, stats['old_workers']) - metrics[:puma_phase].set(labels, stats['phase']) end def set_worker_metrics(stats, labels = {}) diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index eef802caabb..3bfa3da35e0 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -6,8 +6,12 @@ module Gitlab module Metrics module Samplers class RubySampler < BaseSampler + GC_REPORT_BUCKETS = [0.001, 0.002, 0.005, 0.01, 0.05, 0.1, 0.5].freeze + def initialize(interval) - metrics[:process_start_time_seconds].set(labels.merge(worker_label), Time.now.to_i) + GC::Profiler.clear + + metrics[:process_start_time_seconds].set(labels, Time.now.to_i) super end @@ -30,18 +34,18 @@ module Gitlab def init_metrics metrics = { - file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), - memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum), + file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels), + memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels), process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), - process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels), process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), - total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + gc_duration_seconds: ::Gitlab::Metrics.histogram(with_prefix(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS) } GC.stat.keys.each do |key| - metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) + metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels) end metrics @@ -50,47 +54,41 @@ module Gitlab def sample start_time = System.monotonic_time - metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) - metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time) - metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors) + metrics[:file_descriptors].set(labels, System.file_descriptor_count) + metrics[:process_cpu_seconds_total].set(labels, ::Gitlab::Metrics::System.cpu_time) + metrics[:process_max_fds].set(labels, ::Gitlab::Metrics::System.max_open_file_descriptors) set_memory_usage_metrics sample_gc metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) - ensure - GC::Profiler.clear end private def sample_gc - # Collect generic GC stats. + # Observe all GC samples + sample_gc_reports.each do |report| + metrics[:gc_duration_seconds].observe(labels, report[:GC_TIME]) + end + + # Collect generic GC stats GC.stat.each do |key, value| metrics[key].set(labels, value) end + end - # Collect the GC time since last sample in float seconds. - metrics[:total_time].increment(labels, GC::Profiler.total_time) + def sample_gc_reports + GC::Profiler.enable + GC::Profiler.raw_data + ensure + GC::Profiler.clear end def set_memory_usage_metrics memory_usage = System.memory_usage - memory_labels = labels.merge(worker_label) - metrics[:memory_bytes].set(memory_labels, memory_usage) - metrics[:process_resident_memory_bytes].set(memory_labels, memory_usage) - end - - def worker_label - return { worker: 'sidekiq' } if Sidekiq.server? - return {} unless defined?(Unicorn::Worker) - - worker_no = ::Prometheus::Client::Support::Unicorn.worker_id - if worker_no - { worker: worker_no } - else - { worker: 'master' } - end + metrics[:memory_bytes].set(labels, memory_usage) + metrics[:process_resident_memory_bytes].set(labels, memory_usage) end end end diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index 01db507761b..2ee7144fe2f 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -50,7 +50,8 @@ module Gitlab def observe(key, duration) return unless current_transaction - metric_cache_operation_duration_seconds.observe(current_transaction.labels.merge({ operation: key }), duration / 1000.0) + metric_cache_operations_total.increment(current_transaction.labels.merge({ operation: key })) + metric_cache_operation_duration_seconds.observe({ operation: key }, duration / 1000.0) current_transaction.increment(:cache_duration, duration, false) current_transaction.increment(:cache_count, 1, false) current_transaction.increment("cache_#{key}_duration".to_sym, duration, false) @@ -63,12 +64,20 @@ module Gitlab Transaction.current end + def metric_cache_operations_total + @metric_cache_operations_total ||= ::Gitlab::Metrics.counter( + :gitlab_cache_operations_total, + 'Cache operations', + Transaction::BASE_LABELS + ) + end + def metric_cache_operation_duration_seconds @metric_cache_operation_duration_seconds ||= ::Gitlab::Metrics.histogram( :gitlab_cache_operation_duration_seconds, 'Cache access time', - Transaction::BASE_LABELS.merge({ action: nil }), - [0.001, 0.01, 0.1, 1, 10] + {}, + [0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0] ) end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 5c2f07b95e2..51f48095cb5 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -63,10 +63,6 @@ module Gitlab def self.monotonic_time Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) end - - def self.clk_tck - @clk_tck ||= `getconf CLK_TCK`.to_i - end end end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 817db12ac55..53e55269c6e 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -25,7 +25,7 @@ module Gitlab def call if disallowed_request? && Gitlab::Database.read_only? - Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') + Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') # rubocop:disable Gitlab/RailsLogger if json_request? return [403, { 'Content-Type' => APPLICATION_JSON }, [{ 'message' => ERROR_MESSAGE }.to_json]] diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 38b32770e90..c06f106ffe1 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -32,11 +32,6 @@ module Gitlab # Returns the maximum depth starting from the base # A base object with no children has a maximum depth of `1` def max_descendants_depth - unless hierarchy_supported? - # This makes the return value consistent with the case where hierarchy is supported - return descendants_base.exists? ? 1 : nil - end - base_and_descendants(with_depth: true).maximum(DEPTH_COLUMN) end @@ -66,8 +61,6 @@ module Gitlab # each parent. # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors(upto: nil, hierarchy_order: nil) - return ancestors_base unless hierarchy_supported? - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order @@ -81,10 +74,6 @@ module Gitlab # When `with_depth` is `true`, a `depth` column is included where it starts with `1` for the base objects # and incremented as we go down the descendant tree def base_and_descendants(with_depth: false) - unless hierarchy_supported? - return with_depth ? descendants_base.select("1 as #{DEPTH_COLUMN}", objects_table[Arel.star]) : descendants_base - end - read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all)) end @@ -112,8 +101,6 @@ module Gitlab # If nested objects are not supported, ancestors_base is returned. # rubocop: disable CodeReuse/ActiveRecord def all_objects - return ancestors_base unless hierarchy_supported? - ancestors = base_and_ancestors_cte descendants = base_and_descendants_cte @@ -135,10 +122,6 @@ module Gitlab private - def hierarchy_supported? - Gitlab::Database.postgresql? - end - # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors_cte(stop_id = nil, hierarchy_order = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) diff --git a/lib/gitlab/octokit/middleware.rb b/lib/gitlab/octokit/middleware.rb new file mode 100644 index 00000000000..2dd7d08a58b --- /dev/null +++ b/lib/gitlab/octokit/middleware.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Octokit + class Middleware + def initialize(app) + @app = app + end + + def call(env) + Gitlab::UrlBlocker.validate!(env[:url], { allow_localhost: allow_local_requests?, allow_local_network: allow_local_requests? }) + + @app.call(env) + end + + private + + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + end + end +end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 2a2083ebae0..ad1377a0892 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -52,6 +52,16 @@ module Gitlab args[:strategy_class] = args[:strategy_class].constantize end + # Providers that are known to depend on rack-oauth2, like those using + # Omniauth::Strategies::OpenIDConnect, need to be quirked so the + # client_auth_method argument value is passed as a symbol. + if (args[:strategy_class] == OmniAuth::Strategies::OpenIDConnect || + args[:name] == 'openid_connect') && + args[:client_auth_method].is_a?(String) + + args[:client_auth_method] = args[:client_auth_method].to_sym + end + args end diff --git a/lib/gitlab/pages_client.rb b/lib/gitlab/pages_client.rb index d74fdba2241..281eafb142f 100644 --- a/lib/gitlab/pages_client.rb +++ b/lib/gitlab/pages_client.rb @@ -110,7 +110,7 @@ module Gitlab end rescue Errno::EACCES => ex # TODO stop rescuing this exception in GitLab 11.0 https://gitlab.com/gitlab-org/gitlab-ce/issues/45672 - Rails.logger.error("Could not write pages admin token file: #{ex}") + Rails.logger.error("Could not write pages admin token file: #{ex}") # rubocop:disable Gitlab/RailsLogger rescue Errno::EEXIST # Another process wrote the token file concurrently with us. Use their token, not ours. end diff --git a/lib/gitlab/patch/active_record_query_cache.rb b/lib/gitlab/patch/active_record_query_cache.rb new file mode 100644 index 00000000000..71d66bdbe02 --- /dev/null +++ b/lib/gitlab/patch/active_record_query_cache.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# Fixes a bug where the query cache isn't aware of the shared +# ActiveRecord connection used in tests +# https://github.com/rails/rails/issues/36587 + +# To be removed with https://gitlab.com/gitlab-org/gitlab-ce/issues/64413 + +module Gitlab + module Patch + module ActiveRecordQueryCache + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def enable_query_cache! + @query_cache_enabled[connection_cache_key(current_thread)] = true + connection.enable_query_cache! if active_connection? + end + + def disable_query_cache! + @query_cache_enabled.delete connection_cache_key(current_thread) + connection.disable_query_cache! if active_connection? + end + + def query_cache_enabled + @query_cache_enabled[connection_cache_key(current_thread)] + end + + def active_connection? + @thread_cached_conns[connection_cache_key(current_thread)] + end + + private + + def current_thread + @lock_thread || Thread.current + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index a13b3f9e069..f96466b2b00 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -175,6 +175,10 @@ module Gitlab @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze end + def project_wiki_git_route_regex + @project_wiki_git_route_regex ||= /#{PATH_REGEX_STR}\.wiki/.freeze + end + def full_namespace_path_regex @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z} end diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb deleted file mode 100644 index 3a27e26eaba..00000000000 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -# Inspired by https://github.com/peek/peek-pg/blob/master/lib/peek/views/pg.rb -# PEEK_DB_CLIENT is a constant set in config/initializers/peek.rb -module Gitlab - module PerformanceBar - module PeekQueryTracker - def sorted_queries - PEEK_DB_CLIENT.query_details - .sort { |a, b| b[:duration] <=> a[:duration] } - end - - def results - super.merge(queries: sorted_queries) - end - - private - - def setup_subscribers - super - - # Reset each counter when a new request starts - before_request do - PEEK_DB_CLIENT.query_details = [] - end - - subscribe('sql.active_record') do |_, start, finish, _, data| - if Gitlab::SafeRequestStore.store[:peek_enabled] - unless data[:cached] - backtrace = Gitlab::Profiler.clean_backtrace(caller) - track_query(data[:sql].strip, data[:binds], backtrace, start, finish) - end - end - end - end - - def track_query(raw_query, bindings, backtrace, start, finish) - duration = (finish - start) * 1000.0 - query_info = { duration: duration.round(3), sql: raw_query, backtrace: backtrace } - - PEEK_DB_CLIENT.query_details << query_info - end - end - end -end diff --git a/lib/gitlab/phabricator_import/cache/map.rb b/lib/gitlab/phabricator_import/cache/map.rb index fa8b37b20ca..6a2841b6a8e 100644 --- a/lib/gitlab/phabricator_import/cache/map.rb +++ b/lib/gitlab/phabricator_import/cache/map.rb @@ -9,9 +9,15 @@ module Gitlab def get_gitlab_model(phabricator_id) cached_info = get(phabricator_id) - return unless cached_info[:classname] && cached_info[:database_id] - cached_info[:classname].constantize.find_by_id(cached_info[:database_id]) + if cached_info[:classname] && cached_info[:database_id] + object = cached_info[:classname].constantize.find_by_id(cached_info[:database_id]) + else + object = yield if block_given? + set_gitlab_model(object, phabricator_id) if object + end + + object end def set_gitlab_model(object, phabricator_id) diff --git a/lib/gitlab/phabricator_import/conduit/user.rb b/lib/gitlab/phabricator_import/conduit/user.rb new file mode 100644 index 00000000000..fc8c3f7cde9 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/user.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class User + MAX_PAGE_SIZE = 100 + + def initialize(phabricator_url:, api_token:) + @client = Client.new(phabricator_url, api_token) + end + + def users(phids) + phids.each_slice(MAX_PAGE_SIZE).map { |limited_phids| get_page(limited_phids) } + end + + private + + def get_page(phids) + UsersResponse.new(get_users(phids)) + end + + def get_users(phids) + client.get('user.search', + params: { constraints: { phids: phids } }) + end + + attr_reader :client + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/users_response.rb b/lib/gitlab/phabricator_import/conduit/users_response.rb new file mode 100644 index 00000000000..3dfb29a7be5 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/users_response.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + module Conduit + class UsersResponse + def initialize(conduit_response) + @conduit_response = conduit_response + end + + def users + @users ||= conduit_response.data.map do |user_json| + Gitlab::PhabricatorImport::Representation::User.new(user_json) + end + end + + private + + attr_reader :conduit_response + end + end + end +end diff --git a/lib/gitlab/phabricator_import/issues/task_importer.rb b/lib/gitlab/phabricator_import/issues/task_importer.rb index 40d4392cbc1..77ee11c7cdd 100644 --- a/lib/gitlab/phabricator_import/issues/task_importer.rb +++ b/lib/gitlab/phabricator_import/issues/task_importer.rb @@ -8,9 +8,7 @@ module Gitlab end def execute - # TODO: get the user from the project namespace from the username loaded by Phab-id - # https://gitlab.com/gitlab-org/gitlab-ce/issues/60565 - issue.author = User.ghost + issue.author = user_finder.find(task.author_phid) || User.ghost # TODO: Reformat the description with attachments, escaping accidental # links and add attachments @@ -19,6 +17,10 @@ module Gitlab save! + if owner = user_finder.find(task.owner_phid) + issue.assignees << owner + end + issue end @@ -41,6 +43,10 @@ module Gitlab project.issues.new end + def user_finder + @issue_finder ||= Gitlab::PhabricatorImport::UserFinder.new(project, task.phids) + end + def find_issue_by_phabricator_id(phabricator_id) object_map.get_gitlab_model(phabricator_id) end diff --git a/lib/gitlab/phabricator_import/representation/task.rb b/lib/gitlab/phabricator_import/representation/task.rb index 6aedc71b626..ba93fb37a8e 100644 --- a/lib/gitlab/phabricator_import/representation/task.rb +++ b/lib/gitlab/phabricator_import/representation/task.rb @@ -11,6 +11,18 @@ module Gitlab json['phid'] end + def author_phid + json['fields']['authorPHID'] + end + + def owner_phid + json['fields']['ownerPHID'] + end + + def phids + @phids ||= [author_phid, owner_phid] + end + def issue_attributes @issue_attributes ||= { title: issue_title, diff --git a/lib/gitlab/phabricator_import/representation/user.rb b/lib/gitlab/phabricator_import/representation/user.rb new file mode 100644 index 00000000000..7fd7cecc6ae --- /dev/null +++ b/lib/gitlab/phabricator_import/representation/user.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + module Representation + class User + def initialize(json) + @json = json + end + + def phabricator_id + json['phid'] + end + + def username + json['fields']['username'] + end + + private + + attr_reader :json + end + end + end +end diff --git a/lib/gitlab/phabricator_import/user_finder.rb b/lib/gitlab/phabricator_import/user_finder.rb new file mode 100644 index 00000000000..4b50431e0e0 --- /dev/null +++ b/lib/gitlab/phabricator_import/user_finder.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + class UserFinder + def initialize(project, phids) + @project, @phids = project, phids + @loaded_phids = Set.new + end + + def find(phid) + found_user = object_map.get_gitlab_model(phid) do + find_user_for_phid(phid) + end + + loaded_phids << phid + + found_user + end + + private + + attr_reader :project, :phids, :loaded_phids + + def object_map + @object_map ||= Gitlab::PhabricatorImport::Cache::Map.new(project) + end + + def find_user_for_phid(phid) + phabricator_user = phabricator_users.find { |u| u.phabricator_id == phid } + return unless phabricator_user + + project.authorized_users.find_by_username(phabricator_user.username) + end + + def phabricator_users + @user_responses ||= client.users(users_to_request).flat_map(&:users) + end + + def users_to_request + phids - loaded_phids.to_a + end + + def client + @client ||= + Gitlab::PhabricatorImport::Conduit::User + .new(phabricator_url: project.import_data.data['phabricator_url'], + api_token: project.import_data.credentials[:api_token]) + end + end + end +end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 890228e5e78..425c30d67fe 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -21,6 +21,9 @@ module Gitlab lib/gitlab/profiler.rb lib/gitlab/correlation_id.rb lib/gitlab/webpack/dev_server_middleware.rb + lib/gitlab/sidekiq_status/ + lib/gitlab/sidekiq_logging/ + lib/gitlab/sidekiq_middleware/ ].freeze # Takes a URL to profile (can be a fully-qualified URL, or an absolute path) @@ -94,7 +97,7 @@ module Gitlab attr_reader :load_times_by_model, :private_token def debug(message, *) - message.gsub!(private_token, FILTERED_STRING) if private_token + message = message.gsub(private_token, FILTERED_STRING) if private_token _, type, time = *message.match(/(\w+) Load \(([0-9.]+)ms\)/) @@ -166,7 +169,7 @@ module Gitlab [model, times.count, times.sum] end - summarised_load_times.sort_by(&:last).reverse.each do |(model, query_count, time)| + summarised_load_times.sort_by(&:last).reverse_each do |(model, query_count, time)| logger.info("#{model} total (#{query_count}): #{time.round(2)}ms") end end diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb new file mode 100644 index 00000000000..a9270cd536e --- /dev/null +++ b/lib/gitlab/project_authorizations.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +# This class relies on Common Table Expressions to efficiently get all data, +# including data for nested groups. +module Gitlab + class ProjectAuthorizations + attr_reader :user + + # user - The User object for which to calculate the authorizations. + def initialize(user) + @user = user + end + + def calculate + cte = recursive_cte + cte_alias = cte.table.alias(Group.table_name) + projects = Project.arel_table + links = ProjectGroupLink.arel_table + + relations = [ + # The project a user has direct access to. + user.projects.select_for_project_authorization, + + # The personal projects of the user. + user.personal_projects.select_as_maintainer_for_project_authorization, + + # Projects that belong directly to any of the groups the user has + # access to. + Namespace + .unscoped + .select([alias_as_column(projects[:id], 'project_id'), + cte_alias[:access_level]]) + .from(cte_alias) + .joins(:projects), + + # Projects shared with any of the namespaces the user has access to. + Namespace + .unscoped + .select([ + links[:project_id], + least(cte_alias[:access_level], links[:group_access], 'access_level') + ]) + .from(cte_alias) + .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') + .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') + .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') + .where('p_ns.share_with_group_lock IS FALSE') + ] + + ProjectAuthorization + .unscoped + .with + .recursive(cte.to_arel) + .select_from_union(relations) + end + + private + + # Builds a recursive CTE that gets all the groups the current user has + # access to, including any nested groups. + def recursive_cte + cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) + members = Member.arel_table + namespaces = Namespace.arel_table + + # Namespaces the user is a member of. + cte << user.groups + .select([namespaces[:id], members[:access_level]]) + .except(:order) + + # Sub groups of any groups the user is a member of. + cte << Group.select([ + namespaces[:id], + greatest(members[:access_level], cte.table[:access_level], 'access_level') + ]) + .joins(join_cte(cte)) + .joins(join_members) + .except(:order) + + cte + end + + # Builds a LEFT JOIN to join optional memberships onto the CTE. + def join_members + members = Member.arel_table + namespaces = Namespace.arel_table + + cond = members[:source_id] + .eq(namespaces[:id]) + .and(members[:source_type].eq('Namespace')) + .and(members[:requested_at].eq(nil)) + .and(members[:user_id].eq(user.id)) + + Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) + end + + # Builds an INNER JOIN to join namespaces onto the CTE. + def join_cte(cte) + namespaces = Namespace.arel_table + cond = cte.table[:id].eq(namespaces[:parent_id]) + + Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) + end + + def greatest(left, right, column_alias) + sql_function('GREATEST', [left, right], column_alias) + end + + def least(left, right, column_alias) + sql_function('LEAST', [left, right], column_alias) + end + + def sql_function(name, args, column_alias) + alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) + end + + def alias_as_column(value, alias_to) + Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) + end + end +end diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb deleted file mode 100644 index 2372a316ab0..00000000000 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ /dev/null @@ -1,125 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ProjectAuthorizations - # Calculating new project authorizations when supporting nested groups. - # - # This class relies on Common Table Expressions to efficiently get all data, - # including data for nested groups. As a result this class can only be used - # on PostgreSQL. - class WithNestedGroups - attr_reader :user - - # user - The User object for which to calculate the authorizations. - def initialize(user) - @user = user - end - - def calculate - cte = recursive_cte - cte_alias = cte.table.alias(Group.table_name) - projects = Project.arel_table - links = ProjectGroupLink.arel_table - - relations = [ - # The project a user has direct access to. - user.projects.select_for_project_authorization, - - # The personal projects of the user. - user.personal_projects.select_as_maintainer_for_project_authorization, - - # Projects that belong directly to any of the groups the user has - # access to. - Namespace - .unscoped - .select([alias_as_column(projects[:id], 'project_id'), - cte_alias[:access_level]]) - .from(cte_alias) - .joins(:projects), - - # Projects shared with any of the namespaces the user has access to. - Namespace - .unscoped - .select([links[:project_id], - least(cte_alias[:access_level], - links[:group_access], - 'access_level')]) - .from(cte_alias) - .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') - .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') - .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') - .where('p_ns.share_with_group_lock IS FALSE') - ] - - ProjectAuthorization - .unscoped - .with - .recursive(cte.to_arel) - .select_from_union(relations) - end - - private - - # Builds a recursive CTE that gets all the groups the current user has - # access to, including any nested groups. - def recursive_cte - cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) - members = Member.arel_table - namespaces = Namespace.arel_table - - # Namespaces the user is a member of. - cte << user.groups - .select([namespaces[:id], members[:access_level]]) - .except(:order) - - # Sub groups of any groups the user is a member of. - cte << Group.select([namespaces[:id], - greatest(members[:access_level], - cte.table[:access_level], 'access_level')]) - .joins(join_cte(cte)) - .joins(join_members) - .except(:order) - - cte - end - - # Builds a LEFT JOIN to join optional memberships onto the CTE. - def join_members - members = Member.arel_table - namespaces = Namespace.arel_table - - cond = members[:source_id] - .eq(namespaces[:id]) - .and(members[:source_type].eq('Namespace')) - .and(members[:requested_at].eq(nil)) - .and(members[:user_id].eq(user.id)) - - Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) - end - - # Builds an INNER JOIN to join namespaces onto the CTE. - def join_cte(cte) - namespaces = Namespace.arel_table - cond = cte.table[:id].eq(namespaces[:parent_id]) - - Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) - end - - def greatest(left, right, column_alias) - sql_function('GREATEST', [left, right], column_alias) - end - - def least(left, right, column_alias) - sql_function('LEAST', [left, right], column_alias) - end - - def sql_function(name, args, column_alias) - alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) - end - - def alias_as_column(value, alias_to) - Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) - end - end - end -end diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb deleted file mode 100644 index 50b41b17649..00000000000 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ProjectAuthorizations - # Calculating new project authorizations when not supporting nested groups. - class WithoutNestedGroups - attr_reader :user - - # user - The User object for which to calculate the authorizations. - def initialize(user) - @user = user - end - - def calculate - relations = [ - # Projects the user is a direct member of - user.projects.select_for_project_authorization, - - # Personal projects - user.personal_projects.select_as_maintainer_for_project_authorization, - - # Projects of groups the user is a member of - user.groups_projects.select_for_project_authorization, - - # Projects shared with groups the user is a member of - user.groups.joins(:shared_projects).select_for_project_authorization - ] - - ProjectAuthorization - .unscoped - .select_from_union(relations) - end - end - end -end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 0f3b97e2317..2669adb8455 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -29,6 +29,21 @@ module Gitlab end end + def formatted_count(scope) + case scope + when 'blobs' + blobs_count.to_s + when 'notes' + formatted_limited_count(limited_notes_count) + when 'wiki_blobs' + wiki_blobs_count.to_s + when 'commits' + commits_count.to_s + else + super + end + end + def users super.where(id: @project.team.members) # rubocop:disable CodeReuse/ActiveRecord end @@ -108,7 +123,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def notes_finder(type) - NotesFinder.new(project, @current_user, search: query, target_type: type).execute.user.order('updated_at DESC') + NotesFinder.new(@current_user, search: query, target_type: type, project: project).execute.user.order('updated_at DESC') end # rubocop: enable CodeReuse/ActiveRecord @@ -134,9 +149,11 @@ module Gitlab project.repository.commit(key) if Commit.valid_hash?(key) end + # rubocop: disable CodeReuse/ActiveRecord def project_ids_relation - project + Project.where(id: project).select(:id).reorder(nil) end + # rubocop: enabled CodeReuse/ActiveRecord def filter_milestones_by_project(milestones) return Milestone.none unless Ability.allowed?(@current_user, :read_milestone, @project) diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 99885be8755..fa1d1203842 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -13,11 +13,23 @@ module Gitlab end def archive_path - Rails.root.join("vendor/project_templates/#{name}.tar.gz") + self.class.archive_directory.join(archive_filename) + end + + def archive_filename + "#{name}.tar.gz" end def clone_url - "https://gitlab.com/gitlab-org/project-templates/#{name}.git" + "#{preview}.git" + end + + def project_path + URI.parse(preview).path.sub(%r{\A/}, '') + end + + def uri_encoded_project_path + ERB::Util.url_encode(project_path) end def ==(other) @@ -54,7 +66,7 @@ module Gitlab end def archive_directory - Rails.root.join("vendor_directory/project_templates") + Rails.root.join("vendor/project_templates") end end end diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb index 9cc21129547..ba2d33ee1c1 100644 --- a/lib/gitlab/prometheus/query_variables.rb +++ b/lib/gitlab/prometheus/query_variables.rb @@ -4,12 +4,9 @@ module Gitlab module Prometheus module QueryVariables def self.call(environment) - deployment_platform = environment.deployment_platform - namespace = deployment_platform&.kubernetes_namespace_for(environment.project) || '' - { ci_environment_slug: environment.slug, - kube_namespace: namespace, + kube_namespace: environment.deployment_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"} } end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index f13156f898e..9fefffefcde 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -3,6 +3,7 @@ module Gitlab # Helper methods to interact with Prometheus network services & resources class PrometheusClient + include Gitlab::Utils::StrongMemoize Error = Class.new(StandardError) QueryError = Class.new(Gitlab::PrometheusClient::Error) @@ -14,10 +15,17 @@ module Gitlab # Minimal value of the `step` parameter for `query_range` in seconds. QUERY_RANGE_MIN_STEP = 60 - attr_reader :rest_client, :headers + # Key translation between RestClient and Gitlab::HTTP (HTTParty) + RESTCLIENT_GITLAB_HTTP_KEYMAP = { + ssl_cert_store: :cert_store + }.freeze - def initialize(rest_client) - @rest_client = rest_client + attr_reader :api_url, :options + private :api_url, :options + + def initialize(api_url, options = {}) + @api_url = api_url.chomp('/') + @options = options end def ping @@ -27,14 +35,10 @@ module Gitlab def proxy(type, args) path = api_path(type) get(path, args) - rescue RestClient::ExceptionWithResponse => ex - if ex.response - ex.response - else - raise PrometheusClient::Error, "Network connection error" - end - rescue RestClient::Exception - raise PrometheusClient::Error, "Network connection error" + rescue Gitlab::HTTP::ResponseError => ex + raise PrometheusClient::Error, "Network connection error" unless ex.response && ex.response.try(:code) + + handle_response(ex.response) end def query(query, time: Time.now) @@ -78,50 +82,58 @@ module Gitlab private def api_path(type) - ['api', 'v1', type].join('/') + [api_url, 'api', 'v1', type].join('/') end def json_api_get(type, args = {}) path = api_path(type) response = get(path, args) handle_response(response) - rescue RestClient::ExceptionWithResponse => ex - if ex.response - handle_exception_response(ex.response) - else - raise PrometheusClient::Error, "Network connection error" + rescue Gitlab::HTTP::ResponseError => ex + raise PrometheusClient::Error, "Network connection error" unless ex.response && ex.response.try(:code) + + handle_response(ex.response) + end + + def gitlab_http_key(key) + RESTCLIENT_GITLAB_HTTP_KEYMAP[key] || key + end + + def mapped_options + options.keys.map { |k| [gitlab_http_key(k), options[k]] }.to_h + end + + def http_options + strong_memoize(:http_options) do + { follow_redirects: false }.merge(mapped_options) end - rescue RestClient::Exception - raise PrometheusClient::Error, "Network connection error" end def get(path, args) - rest_client[path].get(params: args) + Gitlab::HTTP.get(path, { query: args }.merge(http_options) ) rescue SocketError - raise PrometheusClient::Error, "Can't connect to #{rest_client.url}" + raise PrometheusClient::Error, "Can't connect to #{api_url}" rescue OpenSSL::SSL::SSLError - raise PrometheusClient::Error, "#{rest_client.url} contains invalid SSL data" + raise PrometheusClient::Error, "#{api_url} contains invalid SSL data" rescue Errno::ECONNREFUSED raise PrometheusClient::Error, 'Connection refused' end def handle_response(response) - json_data = parse_json(response.body) - if response.code == 200 && json_data['status'] == 'success' - json_data['data'] || {} - else - raise PrometheusClient::Error, "#{response.code} - #{response.body}" - end - end + response_code = response.try(:code) + response_body = response.try(:body) + + raise PrometheusClient::Error, "#{response_code} - #{response_body}" unless response_code + + json_data = parse_json(response_body) if [200, 400].include?(response_code) - def handle_exception_response(response) - if response.code == 200 && response['status'] == 'success' - response['data'] || {} - elsif response.code == 400 - json_data = parse_json(response.body) + case response_code + when 200 + json_data['data'] if response['status'] == 'success' + when 400 raise PrometheusClient::QueryError, json_data['error'] || 'Bad data received' else - raise PrometheusClient::Error, "#{response.code} - #{response.body}" + raise PrometheusClient::Error, "#{response_code} - #{response_body}" end end diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 3137676ba4b..682edfc4259 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -4,7 +4,14 @@ module Gitlab class PushOptions VALID_OPTIONS = HashWithIndifferentAccess.new({ merge_request: { - keys: [:create, :merge_when_pipeline_succeeds, :target] + keys: [ + :create, + :description, + :merge_when_pipeline_succeeds, + :remove_source_branch, + :target, + :title + ] }, ci: { keys: [:skip] diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index 93030fd454e..ebdae139315 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -3,8 +3,8 @@ module Gitlab module QuickActions class CommandDefinition - attr_accessor :name, :aliases, :description, :explanation, :params, - :condition_block, :parse_params_block, :action_block, :warning, :types + attr_accessor :name, :aliases, :description, :explanation, :execution_message, + :params, :condition_block, :parse_params_block, :action_block, :warning, :types def initialize(name, attributes = {}) @name = name @@ -13,6 +13,7 @@ module Gitlab @description = attributes[:description] || '' @warning = attributes[:warning] || '' @explanation = attributes[:explanation] || '' + @execution_message = attributes[:execution_message] || '' @params = attributes[:params] || [] @condition_block = attributes[:condition_block] @parse_params_block = attributes[:parse_params_block] @@ -48,13 +49,23 @@ module Gitlab end def execute(context, arg) - return if noop? || !available?(context) + return unless executable?(context) count_commands_executed_in(context) execute_block(action_block, context, arg) end + def execute_message(context, arg) + return unless executable?(context) + + if execution_message.respond_to?(:call) + execute_block(execution_message, context, arg) + else + execution_message + end + end + def to_h(context) desc = description if desc.respond_to?(:call) @@ -77,6 +88,10 @@ module Gitlab private + def executable?(context) + !noop? && available?(context) + end + def count_commands_executed_in(context) return unless context.respond_to?(:commands_executed_count=) diff --git a/lib/gitlab/quick_actions/commit_actions.rb b/lib/gitlab/quick_actions/commit_actions.rb index 1018910e8e9..49f5ddf24eb 100644 --- a/lib/gitlab/quick_actions/commit_actions.rb +++ b/lib/gitlab/quick_actions/commit_actions.rb @@ -16,6 +16,13 @@ module Gitlab _("Tags this commit to %{tag_name}.") % { tag_name: tag_name } end end + execution_message do |tag_name, message| + if message.present? + _("Tagged this commit to %{tag_name} with \"%{message}\".") % { tag_name: tag_name, message: message } + else + _("Tagged this commit to %{tag_name}.") % { tag_name: tag_name } + end + end params 'v1.2.3 <message>' parse_params do |tag_name_and_message| tag_name_and_message.split(' ', 2) diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index ecb2169151e..5abbd377642 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -66,6 +66,35 @@ module Gitlab @explanation = block_given? ? block : text end + # Allows to provide a message about quick action execution result, success or failure. + # This message is shown after quick action execution and after saving the note. + # + # Example: + # + # execution_message do |arguments| + # "Added label(s) #{arguments.join(' ')}" + # end + # command :command_key do |arguments| + # # Awesome code block + # end + # + # Note: The execution_message won't be executed unless the condition block returns true. + # execution_message block is executed always after the command block has run, + # for this reason if the condition block doesn't return true after the command block has + # run you need to set the @execution_message variable inside the command block instead as + # shown in the following example. + # + # Example using instance variable: + # + # command :command_key do |arguments| + # # Awesome code block + # @execution_message[:command_key] = 'command_key executed successfully' + # end + # + def execution_message(text = '', &block) + @execution_message = block_given? ? block : text + end + # Allows to define type(s) that must be met in order for the command # to be returned by `.command_names` & `.command_definitions`. # @@ -121,10 +150,16 @@ module Gitlab # comment. # It accepts aliases and takes a block. # + # You can also set the @execution_message instance variable, on conflicts with + # execution_message method the instance variable has precedence. + # # Example: # # command :my_command, :alias_for_my_command do |arguments| # # Awesome code block + # @updates[:my_command] = 'foo' + # + # @execution_message[:my_command] = 'my_command executed successfully' # end def command(*command_names, &block) define_command(CommandDefinition, *command_names, &block) @@ -158,6 +193,7 @@ module Gitlab description: @description, warning: @warning, explanation: @explanation, + execution_message: @execution_message, params: @params, condition_block: @condition_block, parse_params_block: @parse_params_block, @@ -173,6 +209,7 @@ module Gitlab @description = nil @explanation = nil + @execution_message = nil @params = nil @condition_block = nil @warning = nil diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index f7f89d4e897..e5d99ebee35 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -12,10 +12,16 @@ module Gitlab included do # Issue, MergeRequest, Epic: quick actions definitions desc do - "Close this #{quick_action_target.to_ability_name.humanize(capitalize: false)}" + _('Close this %{quick_action_target}') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end explanation do - "Closes this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + _('Closes this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + end + execution_message do + _('Closed this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end types Issuable condition do @@ -28,10 +34,16 @@ module Gitlab end desc do - "Reopen this #{quick_action_target.to_ability_name.humanize(capitalize: false)}" + _('Reopen this %{quick_action_target}') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end explanation do - "Reopens this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + _('Reopens this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + end + execution_message do + _('Reopened this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end types Issuable condition do @@ -45,7 +57,10 @@ module Gitlab desc _('Change title') explanation do |title_param| - _("Changes the title to \"%{title_param}\".") % { title_param: title_param } + _('Changes the title to "%{title_param}".') % { title_param: title_param } + end + execution_message do |title_param| + _('Changed the title to "%{title_param}".') % { title_param: title_param } end params '<New title>' types Issuable @@ -61,7 +76,10 @@ module Gitlab explanation do |labels_param| labels = find_label_references(labels_param) - "Adds #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? + if labels.any? + _("Adds %{labels} %{label_text}.") % + { labels: labels.join(' '), label_text: 'label'.pluralize(labels.count) } + end end params '~label1 ~"label 2"' types Issuable @@ -71,21 +89,15 @@ module Gitlab find_labels.any? end command :label do |labels_param| - label_ids = find_label_ids(labels_param) - - if label_ids.any? - @updates[:add_label_ids] ||= [] - @updates[:add_label_ids] += label_ids - - @updates[:add_label_ids].uniq! - end + run_label_command(labels: find_labels(labels_param), command: :label, updates_key: :add_label_ids) end desc _('Remove all or specific label(s)') explanation do |labels_param = nil| - if labels_param.present? - labels = find_label_references(labels_param) - "Removes #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? + label_references = labels_param.present? ? find_label_references(labels_param) : [] + if label_references.any? + _("Removes %{label_references} %{label_text}.") % + { label_references: label_references.join(' '), label_text: 'label'.pluralize(label_references.count) } else _('Removes all labels.') end @@ -99,7 +111,9 @@ module Gitlab end command :unlabel do |labels_param = nil| if labels_param.present? - label_ids = find_label_ids(labels_param) + labels = find_labels(labels_param) + label_ids = labels.map(&:id) + label_references = labels_to_reference(labels, :name) if label_ids.any? @updates[:remove_label_ids] ||= [] @@ -109,7 +123,10 @@ module Gitlab end else @updates[:label_ids] = [] + label_references = [] end + + @execution_message[:unlabel] = remove_label_message(label_references) end desc _('Replace all label(s)') @@ -125,18 +142,12 @@ module Gitlab current_user.can?(:"admin_#{quick_action_target.to_ability_name}", parent) end command :relabel do |labels_param| - label_ids = find_label_ids(labels_param) - - if label_ids.any? - @updates[:label_ids] ||= [] - @updates[:label_ids] += label_ids - - @updates[:label_ids].uniq! - end + run_label_command(labels: find_labels(labels_param), command: :relabel, updates_key: :label_ids) end - desc _('Add a todo') - explanation _('Adds a todo.') + desc _('Add a To Do') + explanation _('Adds a To Do.') + execution_message _('Added a To Do.') types Issuable condition do quick_action_target.persisted? && @@ -146,8 +157,9 @@ module Gitlab @updates[:todo_event] = 'add' end - desc _('Mark to do as done') - explanation _('Marks to do as done.') + desc _('Mark To Do as done') + explanation _('Marks To Do as done.') + execution_message _('Marked To Do as done.') types Issuable condition do quick_action_target.persisted? && @@ -159,7 +171,12 @@ module Gitlab desc _('Subscribe') explanation do - "Subscribes to this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + _('Subscribes to this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + end + execution_message do + _('Subscribed to this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end types Issuable condition do @@ -172,7 +189,12 @@ module Gitlab desc _('Unsubscribe') explanation do - "Unsubscribes from this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + _('Unsubscribes from this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + end + execution_message do + _('Unsubscribed from this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end types Issuable condition do @@ -187,6 +209,9 @@ module Gitlab explanation do |name| _("Toggles :%{name}: emoji award.") % { name: name } if name end + execution_message do |name| + _("Toggled :%{name}: emoji award.") % { name: name } if name + end params ':emoji:' types Issuable condition do @@ -215,6 +240,41 @@ module Gitlab substitution :tableflip do |comment| "#{comment} #{TABLEFLIP}" end + + private + + def run_label_command(labels:, command:, updates_key:) + return if labels.empty? + + @updates[updates_key] ||= [] + @updates[updates_key] += labels.map(&:id) + @updates[updates_key].uniq! + + label_references = labels_to_reference(labels, :name) + @execution_message[command] = case command + when :relabel + _('Replaced all labels with %{label_references} %{label_text}.') % + { + label_references: label_references.join(' '), + label_text: 'label'.pluralize(label_references.count) + } + when :label + _('Added %{label_references} %{label_text}.') % + { + label_references: label_references.join(' '), + label_text: 'label'.pluralize(labels.count) + } + end + end + + def remove_label_message(label_references) + if label_references.any? + _("Removed %{label_references} %{label_text}.") % + { label_references: label_references.join(' '), label_text: 'label'.pluralize(label_references.count) } + else + _('Removed all labels.') + end + end end end end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 85e62f950c8..da28fbf5be0 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -12,6 +12,9 @@ module Gitlab explanation do |due_date| _("Sets the due date to %{due_date}.") % { due_date: due_date.strftime('%b %-d, %Y') } if due_date end + execution_message do |due_date| + _("Set the due date to %{due_date}.") % { due_date: due_date.strftime('%b %-d, %Y') } if due_date + end params '<in 2 days | this Friday | December 31st>' types Issue condition do @@ -27,6 +30,7 @@ module Gitlab desc _('Remove due date') explanation _('Removes the due date.') + execution_message _('Removed the due date.') types Issue condition do quick_action_target.persisted? && @@ -49,22 +53,26 @@ module Gitlab current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) && quick_action_target.project.boards.count == 1 end - # rubocop: disable CodeReuse/ActiveRecord command :board_move do |target_list_name| - label_ids = find_label_ids(target_list_name) + labels = find_labels(target_list_name) + label_ids = labels.map(&:id) - if label_ids.size == 1 + if label_ids.size > 1 + message = _('Failed to move this issue because only a single label can be provided.') + elsif !Label.on_project_board?(quick_action_target.project_id, label_ids.first) + message = _('Failed to move this issue because label was not found.') + else label_id = label_ids.first - # Ensure this label corresponds to a list on the board - next unless Label.on_project_boards(quick_action_target.project_id).where(id: label_id).exists? - @updates[:remove_label_ids] = - quick_action_target.labels.on_project_boards(quick_action_target.project_id).where.not(id: label_id).pluck(:id) + quick_action_target.labels.on_project_boards(quick_action_target.project_id).where.not(id: label_id).pluck(:id) # rubocop: disable CodeReuse/ActiveRecord @updates[:add_label_ids] = [label_id] + + message = _("Moved issue to %{label} column in the board.") % { label: labels_to_reference(labels).first } end + + @execution_message[:board_move] = message end - # rubocop: enable CodeReuse/ActiveRecord desc _('Mark this issue as a duplicate of another issue') explanation do |duplicate_reference| @@ -81,7 +89,13 @@ module Gitlab if canonical_issue.present? @updates[:canonical_issue_id] = canonical_issue.id + + message = _("Marked this issue as a duplicate of %{duplicate_param}.") % { duplicate_param: duplicate_param } + else + message = _('Failed to mark this issue as a duplicate because referenced issue was not found.') end + + @execution_message[:duplicate] = message end desc _('Move this issue to another project.') @@ -99,12 +113,21 @@ module Gitlab if target_project.present? @updates[:target_project] = target_project + + message = _("Moved this issue to %{path_to_project}.") % { path_to_project: target_project_path } + else + message = _("Failed to move this issue because target project doesn't exist.") end + + @execution_message[:move] = message end - desc _('Make issue confidential.') + desc _('Make issue confidential') explanation do - _('Makes this issue confidential') + _('Makes this issue confidential.') + end + execution_message do + _('Made this issue confidential.') end types Issue condition do @@ -114,12 +137,19 @@ module Gitlab @updates[:confidential] = true end - desc _('Create a merge request.') + desc _('Create a merge request') explanation do |branch_name = nil| if branch_name - _("Creates branch '%{branch_name}' and a merge request to resolve this issue") % { branch_name: branch_name } + _("Creates branch '%{branch_name}' and a merge request to resolve this issue.") % { branch_name: branch_name } + else + _('Creates a branch and a merge request to resolve this issue.') + end + end + execution_message do |branch_name = nil| + if branch_name + _("Created branch '%{branch_name}' and a merge request to resolve this issue.") % { branch_name: branch_name } else - "Creates a branch and a merge request to resolve this issue" + _('Created a branch and a merge request to resolve this issue.') end end params "<branch name>" diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index e1579cfddc0..533c74ba9b4 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -9,12 +9,9 @@ module Gitlab included do # Issue, MergeRequest: quick actions definitions desc _('Assign') - # rubocop: disable CodeReuse/ActiveRecord explanation do |users| - users = quick_action_target.allows_multiple_assignees? ? users : users.take(1) - "Assigns #{users.map(&:to_reference).to_sentence}." + _('Assigns %{assignee_users_sentence}.') % { assignee_users_sentence: assignee_users_sentence(users) } end - # rubocop: enable CodeReuse/ActiveRecord params do quick_action_target.allows_multiple_assignees? ? '@user1 @user2' : '@user' end @@ -26,7 +23,10 @@ module Gitlab extract_users(assignee_param) end command :assign do |users| - next if users.empty? + if users.empty? + @execution_message[:assign] = _("Failed to assign a user because no user was found.") + next + end if quick_action_target.allows_multiple_assignees? @updates[:assignee_ids] ||= quick_action_target.assignees.map(&:id) @@ -34,6 +34,8 @@ module Gitlab else @updates[:assignee_ids] = [users.first.id] end + + @execution_message[:assign] = _('Assigned %{assignee_users_sentence}.') % { assignee_users_sentence: assignee_users_sentence(users) } end desc do @@ -44,9 +46,14 @@ module Gitlab end end explanation do |users = nil| - assignees = quick_action_target.assignees - assignees &= users if users.present? && quick_action_target.allows_multiple_assignees? - "Removes #{'assignee'.pluralize(assignees.size)} #{assignees.map(&:to_reference).to_sentence}." + assignees = assignees_for_removal(users) + _("Removes %{assignee_text} %{assignee_references}.") % + { assignee_text: 'assignee'.pluralize(assignees.size), assignee_references: assignees.map(&:to_reference).to_sentence } + end + execution_message do |users = nil| + assignees = assignees_for_removal(users) + _("Removed %{assignee_text} %{assignee_references}.") % + { assignee_text: 'assignee'.pluralize(assignees.size), assignee_references: assignees.map(&:to_reference).to_sentence } end params do quick_action_target.allows_multiple_assignees? ? '@user1 @user2' : '' @@ -74,6 +81,9 @@ module Gitlab explanation do |milestone| _("Sets the milestone to %{milestone_reference}.") % { milestone_reference: milestone.to_reference } if milestone end + execution_message do |milestone| + _("Set the milestone to %{milestone_reference}.") % { milestone_reference: milestone.to_reference } if milestone + end params '%"milestone"' types Issue, MergeRequest condition do @@ -92,6 +102,9 @@ module Gitlab explanation do _("Removes %{milestone_reference} milestone.") % { milestone_reference: quick_action_target.milestone.to_reference(format: :name) } end + execution_message do + _("Removed %{milestone_reference} milestone.") % { milestone_reference: quick_action_target.milestone.to_reference(format: :name) } + end types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -116,17 +129,22 @@ module Gitlab extract_references(issuable_param, :merge_request).first end command :copy_metadata do |source_issuable| - if source_issuable.present? && source_issuable.project.id == quick_action_target.project.id + if can_copy_metadata?(source_issuable) @updates[:add_label_ids] = source_issuable.labels.map(&:id) @updates[:milestone_id] = source_issuable.milestone.id if source_issuable.milestone + + @execution_message[:copy_metadata] = _("Copied labels and milestone from %{source_issuable_reference}.") % { source_issuable_reference: source_issuable.to_reference } end end desc _('Set time estimate') explanation do |time_estimate| - time_estimate = Gitlab::TimeTrackingFormatter.output(time_estimate) - - _("Sets time estimate to %{time_estimate}.") % { time_estimate: time_estimate } if time_estimate + formatted_time_estimate = format_time_estimate(time_estimate) + _("Sets time estimate to %{time_estimate}.") % { time_estimate: formatted_time_estimate } if formatted_time_estimate + end + execution_message do |time_estimate| + formatted_time_estimate = format_time_estimate(time_estimate) + _("Set time estimate to %{time_estimate}.") % { time_estimate: formatted_time_estimate } if formatted_time_estimate end params '<1w 3d 2h 14m>' types Issue, MergeRequest @@ -144,18 +162,12 @@ module Gitlab desc _('Add or subtract spent time') explanation do |time_spent, time_spent_date| - if time_spent - if time_spent > 0 - verb = _('Adds') - value = time_spent - else - verb = _('Subtracts') - value = -time_spent - end - - _("%{verb} %{time_spent_value} spent time.") % { verb: verb, time_spent_value: Gitlab::TimeTrackingFormatter.output(value) } - end + spend_time_message(time_spent, time_spent_date, false) end + execution_message do |time_spent, time_spent_date| + spend_time_message(time_spent, time_spent_date, true) + end + params '<time(1h30m | -1h30m)> <date(YYYY-MM-DD)>' types Issue, MergeRequest condition do @@ -176,6 +188,7 @@ module Gitlab desc _('Remove time estimate') explanation _('Removes time estimate.') + execution_message _('Removed time estimate.') types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -187,6 +200,7 @@ module Gitlab desc _('Remove spent time') explanation _('Removes spent time.') + execution_message _('Removed spent time.') condition do quick_action_target.persisted? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) @@ -197,7 +211,8 @@ module Gitlab end desc _("Lock the discussion") - explanation _("Locks the discussion") + explanation _("Locks the discussion.") + execution_message _("Locked the discussion.") types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -209,7 +224,8 @@ module Gitlab end desc _("Unlock the discussion") - explanation _("Unlocks the discussion") + explanation _("Unlocks the discussion.") + execution_message _("Unlocked the discussion.") types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -219,6 +235,47 @@ module Gitlab command :unlock do @updates[:discussion_locked] = false end + + private + + def assignee_users_sentence(users) + if quick_action_target.allows_multiple_assignees? + users + else + [users.first] + end.map(&:to_reference).to_sentence + end + + def assignees_for_removal(users) + assignees = quick_action_target.assignees + if users.present? && quick_action_target.allows_multiple_assignees? + assignees & users + else + assignees + end + end + + def can_copy_metadata?(source_issuable) + source_issuable.present? && source_issuable.project_id == quick_action_target.project_id + end + + def format_time_estimate(time_estimate) + Gitlab::TimeTrackingFormatter.output(time_estimate) + end + + def spend_time_message(time_spent, time_spent_date, paste_tense) + return unless time_spent + + if time_spent > 0 + verb = paste_tense ? _('Added') : _('Adds') + value = time_spent + else + verb = paste_tense ? _('Subtracted') : _('Subtracts') + value = -time_spent + end + + _("%{verb} %{time_spent_value} spent time.") % { verb: verb, time_spent_value: format_time_estimate(value) } + end end end end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index bade59182a1..e9127095a0d 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -8,8 +8,9 @@ module Gitlab included do # MergeRequest only quick actions definitions - desc 'Merge (when the pipeline succeeds)' - explanation 'Merges this merge request when the pipeline succeeds.' + desc _('Merge (when the pipeline succeeds)') + explanation _('Merges this merge request when the pipeline succeeds.') + execution_message _('Scheduled to merge this merge request when the pipeline succeeds.') types MergeRequest condition do last_diff_sha = params && params[:merge_request_diff_head_sha] @@ -22,10 +23,22 @@ module Gitlab desc 'Toggle the Work In Progress status' explanation do - verb = quick_action_target.work_in_progress? ? 'Unmarks' : 'Marks' noun = quick_action_target.to_ability_name.humanize(capitalize: false) - "#{verb} this #{noun} as Work In Progress." + if quick_action_target.work_in_progress? + _("Unmarks this %{noun} as Work In Progress.") + else + _("Marks this %{noun} as Work In Progress.") + end % { noun: noun } end + execution_message do + noun = quick_action_target.to_ability_name.humanize(capitalize: false) + if quick_action_target.work_in_progress? + _("Unmarked this %{noun} as Work In Progress.") + else + _("Marked this %{noun} as Work In Progress.") + end % { noun: noun } + end + types MergeRequest condition do quick_action_target.respond_to?(:work_in_progress?) && @@ -36,9 +49,12 @@ module Gitlab @updates[:wip_event] = quick_action_target.work_in_progress? ? 'unwip' : 'wip' end - desc 'Set target branch' + desc _('Set target branch') explanation do |branch_name| - "Sets target branch to #{branch_name}." + _('Sets target branch to %{branch_name}.') % { branch_name: branch_name } + end + execution_message do |branch_name| + _('Set target branch to %{branch_name}.') % { branch_name: branch_name } end params '<Local branch name>' types MergeRequest diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb index 2f78ea05cf0..0fda056a4fe 100644 --- a/lib/gitlab/quick_actions/substitution_definition.rb +++ b/lib/gitlab/quick_actions/substitution_definition.rb @@ -17,8 +17,9 @@ module Gitlab return unless content all_names.each do |a_name| - content.gsub!(%r{/#{a_name} ?(.*)$}i, execute_block(action_block, context, '\1')) + content = content.gsub(%r{/#{a_name} ?(.*)$}i, execute_block(action_block, context, '\1')) end + content end end diff --git a/lib/gitlab/reference_counter.rb b/lib/gitlab/reference_counter.rb index d2dbc6f5ef5..1c43de35816 100644 --- a/lib/gitlab/reference_counter.rb +++ b/lib/gitlab/reference_counter.rb @@ -22,6 +22,7 @@ module Gitlab end end + # rubocop:disable Gitlab/RailsLogger def decrease redis_cmd do |redis| current_value = redis.decr(key) @@ -32,6 +33,7 @@ module Gitlab end end end + # rubocop:enable Gitlab/RailsLogger private @@ -39,7 +41,7 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| yield(redis) } true rescue => e - Rails.logger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") + Rails.logger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") # rubocop:disable Gitlab/RailsLogger false end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 7a1a2eaf6c0..e6372a42dda 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -4,14 +4,6 @@ module Gitlab module Regex extend self - def namespace_name_regex - @namespace_name_regex ||= /\A[\p{Alnum}\p{Pd}_\. ]*\z/.freeze - end - - def namespace_name_regex_message - "can contain only letters, digits, '_', '.', dash and space." - end - def project_name_regex @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9c0}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9c0}_\. ]*\z/.freeze end @@ -54,6 +46,18 @@ module Gitlab "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', and spaces, but it cannot start or end with '/'" end + def environment_scope_regex_chars + "#{environment_name_regex_chars}\\*" + end + + def environment_scope_regex + @environment_scope_regex ||= /\A[#{environment_scope_regex_chars}]+\z/.freeze + end + + def environment_scope_regex_message + "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', '*' and spaces" + end + def kubernetes_namespace_regex /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/ end @@ -102,6 +106,12 @@ module Gitlab }mx end + # Based on Jira's project key format + # https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html + def jira_issue_key_regex + @jira_issue_key_regex ||= /[A-Z][A-Z_0-9]+-\d+/ + end + def jira_transition_id_regex @jira_transition_id_regex ||= /\d+/ end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 931298b5117..75503ee1789 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -23,6 +23,37 @@ module Gitlab end end + # Caches and strongly memoizes the method as a Redis Set. + # + # This only works for methods that do not take any arguments. The method + # should return an Array of Strings to be cached. + # + # In addition to overriding the named method, a "name_include?" method is + # defined. This uses the "SISMEMBER" query to efficiently check membership + # without needing to load the entire set into memory. + # + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method_as_redis_set(name, fallback: nil) + uncached_name = alias_uncached_method(name) + + define_method(name) do + cache_method_output_as_redis_set(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + + define_method("#{name}_include?") do |value| + # If the cache isn't populated, we can't rely on it + return redis_set_cache.include?(name, value) if redis_set_cache.exist?(name) + + # Since we have to pull all branch names to populate the cache, use + # the data we already have to answer the query just this once + __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend + end + end + # Caches truthy values from the method. All values are strongly memoized, # and cached in RequestStore. # @@ -84,6 +115,11 @@ module Gitlab raise NotImplementedError end + # RepositorySetCache to be used. Should be overridden by the including class + def redis_set_cache + raise NotImplementedError + end + # List of cached methods. Should be overridden by the including class def cached_methods raise NotImplementedError @@ -100,6 +136,18 @@ module Gitlab end end + # Caches and strongly memoizes the supplied block as a Redis Set. The result + # will be provided as a sorted array. + # + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method_output_as_redis_set(name, fallback: nil, &block) + memoize_method_output(name, fallback: fallback) do + redis_set_cache.fetch(name, &block).sort + end + end + # Caches truthy values from the supplied block. All values are strongly # memoized, and cached in RequestStore. # @@ -145,7 +193,7 @@ module Gitlab def expire_method_caches(methods) methods.each do |name| unless cached_methods.include?(name.to_sym) - Rails.logger.error "Requested to expire non-existent method '#{name}' for Repository" + Rails.logger.error "Requested to expire non-existent method '#{name}' for Repository" # rubocop:disable Gitlab/RailsLogger next end @@ -154,6 +202,7 @@ module Gitlab clear_memoization(memoizable_name(name)) end + expire_redis_set_method_caches(methods) expire_request_store_method_caches(methods) end @@ -169,6 +218,10 @@ module Gitlab end end + def expire_redis_set_method_caches(methods) + methods.each { |name| redis_set_cache.expire(name) } + end + # All cached repository methods depend on the existence of a Git repository, # so if the repository doesn't exist, we already know not to call it. def fallback_early?(method_name) diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb new file mode 100644 index 00000000000..fb634328a95 --- /dev/null +++ b/lib/gitlab/repository_set_cache.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +# Interface to the Redis-backed cache store for keys that use a Redis set +module Gitlab + class RepositorySetCache + attr_reader :repository, :namespace, :expires_in + + def initialize(repository, extra_namespace: nil, expires_in: 2.weeks) + @repository = repository + @namespace = "#{repository.full_path}:#{repository.project.id}" + @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace + @expires_in = expires_in + end + + def cache_key(type) + [type, namespace, 'set'].join(':') + end + + def expire(key) + with { |redis| redis.del(cache_key(key)) } + end + + def exist?(key) + with { |redis| redis.exists(cache_key(key)) } + end + + def read(key) + with { |redis| redis.smembers(cache_key(key)) } + end + + def write(key, value) + full_key = cache_key(key) + + with do |redis| + redis.multi do + redis.del(full_key) + + # Splitting into groups of 1000 prevents us from creating a too-long + # Redis command + value.in_groups_of(1000, false) { |subset| redis.sadd(full_key, subset) } + + redis.expire(full_key, expires_in) + end + end + + value + end + + def fetch(key, &block) + if exist?(key) + read(key) + else + write(key, yield) + end + end + + def include?(key, value) + with { |redis| redis.sismember(cache_key(key), value) } + end + + private + + def with(&blk) + Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb index 64593153686..033e451dbee 100644 --- a/lib/gitlab/request_profiler.rb +++ b/lib/gitlab/request_profiler.rb @@ -6,6 +6,21 @@ module Gitlab module RequestProfiler PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles".freeze + def all + Dir["#{PROFILES_DIR}/*.{html,txt}"].map do |path| + Profile.new(File.basename(path)) + end.select(&:valid?) + end + module_function :all + + def find(name) + file_path = File.join(PROFILES_DIR, name) + return unless File.exist?(file_path) + + Profile.new(name) + end + module_function :find + def profile_token Rails.cache.fetch('profile-token') do Devise.friendly_token diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb index 7615f6f443b..99958d7a211 100644 --- a/lib/gitlab/request_profiler/middleware.rb +++ b/lib/gitlab/request_profiler/middleware.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'ruby-prof' +require 'memory_profiler' module Gitlab module RequestProfiler @@ -28,22 +29,73 @@ module Gitlab end def call_with_profiling(env) + case env['HTTP_X_PROFILE_MODE'] + when 'execution', nil + call_with_call_stack_profiling(env) + when 'memory' + call_with_memory_profiling(env) + else + raise ActionController::BadRequest, invalid_profile_mode(env) + end + end + + def invalid_profile_mode(env) + <<~HEREDOC + Invalid X-Profile-Mode: #{env['HTTP_X_PROFILE_MODE']}. + Supported profile mode request header: + - X-Profile-Mode: execution + - X-Profile-Mode: memory + HEREDOC + end + + def call_with_call_stack_profiling(env) ret = nil - result = RubyProf::Profile.profile do + report = RubyProf::Profile.profile do ret = catch(:warden) do @app.call(env) end end - printer = RubyProf::CallStackPrinter.new(result) - file_name = "#{env['PATH_INFO'].tr('/', '|')}_#{Time.current.to_i}.html" + generate_report(env, 'execution', 'html') do |file| + printer = RubyProf::CallStackPrinter.new(report) + printer.print(file) + end + + handle_request_ret(ret) + end + + def call_with_memory_profiling(env) + ret = nil + report = MemoryProfiler.report do + ret = catch(:warden) do + @app.call(env) + end + end + + generate_report(env, 'memory', 'txt') do |file| + report.pretty_print(to_file: file) + end + + handle_request_ret(ret) + end + + def generate_report(env, report_type, extension) + file_name = "#{env['PATH_INFO'].tr('/', '|')}_#{Time.current.to_i}"\ + "_#{report_type}.#{extension}" file_path = "#{PROFILES_DIR}/#{file_name}" FileUtils.mkdir_p(PROFILES_DIR) - File.open(file_path, 'wb') do |file| - printer.print(file) + + begin + File.open(file_path, 'wb') do |file| + yield(file) + end + rescue + FileUtils.rm(file_path) end + end + def handle_request_ret(ret) if ret.is_a?(Array) ret else diff --git a/lib/gitlab/request_profiler/profile.rb b/lib/gitlab/request_profiler/profile.rb index 46996ef8c51..76c675658b1 100644 --- a/lib/gitlab/request_profiler/profile.rb +++ b/lib/gitlab/request_profiler/profile.rb @@ -3,42 +3,40 @@ module Gitlab module RequestProfiler class Profile - attr_reader :name, :time, :request_path + attr_reader :name, :time, :file_path, :request_path, :profile_mode, :type alias_method :to_param, :name - def self.all - Dir["#{PROFILES_DIR}/*.html"].map do |path| - new(File.basename(path)) - end - end - - def self.find(name) - name_dup = name.dup - name_dup << '.html' unless name.end_with?('.html') - - file_path = "#{PROFILES_DIR}/#{name_dup}" - return unless File.exist?(file_path) - - new(name_dup) - end - def initialize(name) @name = name + @file_path = File.join(PROFILES_DIR, name) set_attributes end - def content - File.read("#{PROFILES_DIR}/#{name}") + def valid? + @request_path.present? + end + + def content_type + case type + when 'html' + 'text/html' + when 'txt' + 'text/plain' + end end private def set_attributes - _, path, timestamp = name.split(/(.*)_(\d+)\.html$/) - @request_path = path.tr('|', '/') - @time = Time.at(timestamp.to_i).utc + matches = name.match(/^(?<path>.*)_(?<timestamp>\d+)(_(?<profile_mode>\w+))?\.(?<type>html|txt)$/) + return unless matches + + @request_path = matches[:path].tr('|', '/') + @time = Time.at(matches[:timestamp].to_i).utc + @profile_mode = matches[:profile_mode] || 'unknown' + @type = matches[:type] end end end diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb new file mode 100644 index 00000000000..8bb8c547ae1 --- /dev/null +++ b/lib/gitlab/rugged_instrumentation.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module RuggedInstrumentation + def self.query_time + SafeRequestStore[:rugged_query_time] ||= 0 + end + + def self.query_time=(duration) + SafeRequestStore[:rugged_query_time] = duration + end + + def self.query_time_ms + (self.query_time * 1000).round(2) + end + + def self.query_count + SafeRequestStore[:rugged_call_count] ||= 0 + end + + def self.increment_query_count + SafeRequestStore[:rugged_call_count] ||= 0 + SafeRequestStore[:rugged_call_count] += 1 + end + + def self.active? + SafeRequestStore.active? + end + + def self.peek_enabled? + SafeRequestStore[:peek_enabled] + end + + def self.add_call_details(details) + return unless peek_enabled? + + Gitlab::SafeRequestStore[:rugged_call_details] ||= [] + Gitlab::SafeRequestStore[:rugged_call_details] << details + end + + def self.list_call_details + return [] unless peek_enabled? + + Gitlab::SafeRequestStore[:rugged_call_details] || [] + end + end +end diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb index 0928ccdc324..bb4e4ce7bbc 100644 --- a/lib/gitlab/sanitizers/exif.rb +++ b/lib/gitlab/sanitizers/exif.rb @@ -48,7 +48,7 @@ module Gitlab attr_reader :logger - def initialize(logger: Rails.logger) + def initialize(logger: Rails.logger) # rubocop:disable Gitlab/RailsLogger @logger = logger end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 7c1e6b1baff..ce4c1611687 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -43,6 +43,29 @@ module Gitlab without_count ? collection.without_count : collection end + def formatted_count(scope) + case scope + when 'projects' + formatted_limited_count(limited_projects_count) + when 'issues' + formatted_limited_count(limited_issues_count) + when 'merge_requests' + formatted_limited_count(limited_merge_requests_count) + when 'milestones' + formatted_limited_count(limited_milestones_count) + when 'users' + formatted_limited_count(limited_users_count) + end + end + + def formatted_limited_count(count) + if count >= COUNT_LIMIT + "#{COUNT_LIMIT - 1}+" + else + count.to_s + end + end + def limited_projects_count @limited_projects_count ||= limited_count(projects) end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 764db14d720..005cb3112b8 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -39,9 +39,14 @@ module Gitlab # development and test. If you need development and test to behave # just the same as production you can use this instead of # track_exception. + # + # If the exception implements the method `sentry_extra_data` and that method + # returns a Hash, then the return value of that method will be merged into + # `extra`. Exceptions can use this mechanism to provide structured data + # to sentry in addition to their message and back-trace. def self.track_acceptable_exception(exception, issue_url: nil, extra: {}) if enabled? - extra[:issue_url] = issue_url if issue_url + extra = build_extra_data(exception, issue_url, extra) context # Make sure we've set everything we know in the context Raven.capture_exception(exception, tags: default_tags, extra: extra) @@ -58,5 +63,15 @@ module Gitlab locale: I18n.locale } end + + def self.build_extra_data(exception, issue_url, extra) + exception.try(:sentry_extra_data)&.tap do |data| + extra.merge!(data) if data.is_a?(Hash) + end + + extra.merge({ issue_url: issue_url }.compact) + end + + private_class_method :build_extra_data end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 93182607616..0fa17b3f559 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -78,7 +78,7 @@ module Gitlab true rescue => err # Once the Rugged codes gets removes this can be improved - Rails.logger.error("Failed to add repository #{storage}/#{disk_path}: #{err}") + Rails.logger.error("Failed to add repository #{storage}/#{disk_path}: #{err}") # rubocop:disable Gitlab/RailsLogger false end @@ -153,7 +153,7 @@ module Gitlab !!rm_directory(storage, "#{name}.git") rescue ArgumentError => e - Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git") + Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git") # rubocop:disable Gitlab/RailsLogger false end @@ -238,7 +238,7 @@ module Gitlab def remove_keys_not_found_in_db return unless self.authorized_keys_enabled? - Rails.logger.info("Removing keys not found in DB") + Rails.logger.info("Removing keys not found in DB") # rubocop:disable Gitlab/RailsLogger batch_read_key_ids do |ids_in_file| ids_in_file.uniq! @@ -248,7 +248,7 @@ module Gitlab ids_to_remove = ids_in_file - keys_in_db.pluck(:id) ids_to_remove.each do |id| - Rails.logger.info("Removing key-#{id} not found in DB") + Rails.logger.info("Removing key-#{id} not found in DB") # rubocop:disable Gitlab/RailsLogger remove_key("key-#{id}") end end @@ -368,7 +368,7 @@ module Gitlab return true if status.zero? - Rails.logger.error("gitlab-shell failed with error #{status}: #{output}") + Rails.logger.error("gitlab-shell failed with error #{status}: #{output}") # rubocop:disable Gitlab/RailsLogger false end @@ -465,7 +465,7 @@ module Gitlab end def logger - Rails.logger + Rails.logger # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 159ce27e702..cbd89b7629f 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -96,12 +96,7 @@ module Gitlab private def raw_explain(query) - explain = - if Gitlab::Database.postgresql? - "EXPLAIN ANALYZE #{query};" - else - "EXPLAIN #{query};" - end + explain = "EXPLAIN ANALYZE #{query};" ActiveRecord::Base.connection.execute(explain) end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index fdc0d518c59..48b1524f9c7 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -8,16 +8,16 @@ module Gitlab MAXIMUM_JOB_ARGUMENTS_LENGTH = 10.kilobytes def call(job, queue) - started_at = current_time + started_time = get_time base_payload = parse_job(job) - Sidekiq.logger.info log_job_start(started_at, base_payload) + Sidekiq.logger.info log_job_start(base_payload) yield - Sidekiq.logger.info log_job_done(started_at, base_payload) + Sidekiq.logger.info log_job_done(job, started_time, base_payload) rescue => job_exception - Sidekiq.logger.warn log_job_done(started_at, base_payload, job_exception) + Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception) raise end @@ -28,17 +28,29 @@ module Gitlab "#{payload['class']} JID-#{payload['jid']}" end - def log_job_start(started_at, payload) + def add_instrumentation_keys!(job, output_payload) + output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper::KEYS)) + end + + def log_job_start(payload) payload['message'] = "#{base_message(payload)}: start" payload['job_status'] = 'start' + # Old gitlab-shell messages don't provide enqueued_at/created_at attributes + enqueued_at = payload['enqueued_at'] || payload['created_at'] + if enqueued_at + payload['scheduling_latency_s'] = elapsed_by_absolute_time(Time.iso8601(enqueued_at)) + end + payload end - def log_job_done(started_at, payload, job_exception = nil) + def log_job_done(job, started_time, payload, job_exception = nil) payload = payload.dup - payload['duration'] = elapsed(started_at) - payload['completed_at'] = Time.now.utc + add_instrumentation_keys!(job, payload) + + elapsed_time = elapsed(started_time) + add_time_keys!(elapsed_time, payload) message = base_message(payload) @@ -58,6 +70,14 @@ module Gitlab payload end + def add_time_keys!(time, payload) + payload['duration'] = time[:duration].round(3) + payload['system_s'] = time[:stime].round(3) + payload['user_s'] = time[:utime].round(3) + payload['child_s'] = time[:ctime].round(3) if time[:ctime] > 0 + payload['completed_at'] = Time.now.utc + end + def parse_job(job) job = job.dup @@ -78,8 +98,29 @@ module Gitlab end end - def elapsed(start) - (current_time - start).round(3) + def elapsed_by_absolute_time(start) + (Time.now.utc - start).to_f.round(3) + end + + def elapsed(t0) + t1 = get_time + { + duration: t1[:now] - t0[:now], + stime: t1[:times][:stime] - t0[:times][:stime], + utime: t1[:times][:utime] - t0[:times][:utime], + ctime: ctime(t1[:times]) - ctime(t0[:times]) + } + end + + def get_time + { + now: current_time, + times: Process.times + } + end + + def ctime(times) + times[:cstime] + times[:cutime] end def current_time diff --git a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb new file mode 100644 index 00000000000..979a3fce7e6 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class InstrumentationLogger + def call(worker, job, queue) + yield + + # The Sidekiq logger is called outside the middleware block, so + # we need to modify the job hash to pass along this information + # since RequestStore is only active in the Sidekiq middleware. + # + # Modifying the job hash in a middleware is permitted by Sidekiq + # because Sidekiq keeps a pristine copy of the original hash + # before sending it to the middleware: + # https://github.com/mperham/sidekiq/blob/53bd529a0c3f901879925b8390353129c465b1f2/lib/sidekiq/processor.rb#L115-L118 + ::Gitlab::InstrumentationHelper.add_instrumentation_data(job) + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 671d795ec33..49c4fdc3033 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -14,9 +14,12 @@ module Gitlab # shut Sidekiq down MUTEX = Mutex.new + attr_reader :worker + def call(worker, job, queue) yield + @worker = worker current_rss = get_rss return unless MAX_RSS > 0 && current_rss > MAX_RSS @@ -25,9 +28,11 @@ module Gitlab # Return if another thread is already waiting to shut Sidekiq down next unless MUTEX.try_lock - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ - " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + warn("Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ + " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}") + + warn("Sidekiq worker PID-#{pid} will stop fetching new jobs"\ + " in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later") # Wait `GRACE_TIME` to give the memory intensive job time to finish. # Then, tell Sidekiq to stop fetching new jobs. @@ -59,24 +64,28 @@ module Gitlab def wait_and_signal_pgroup(time, signal, explanation) return wait_and_signal(time, signal, explanation) unless Process.getpgrp == pid - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + warn("waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) sleep(time) - Sidekiq.logger.warn "sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + warn("sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) Process.kill(signal, 0) end def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + warn("waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) sleep(time) - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + warn("sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) Process.kill(signal, pid) end def pid Process.pid end + + def warn(message, signal: nil) + Sidekiq.logger.warn(class: worker.class.name, pid: pid, signal: signal, message: message) + end end end end diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb new file mode 100644 index 00000000000..3dc9521ee8b --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/metrics.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class Metrics + # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq + # timeframes than the DEFAULT_BUCKET definition. Defined in seconds. + SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze + + def initialize + @metrics = init_metrics + end + + def call(_worker, job, queue) + labels = create_labels(queue) + @metrics[:sidekiq_running_jobs].increment(labels, 1) + + if job['retry_count'].present? + @metrics[:sidekiq_jobs_retried_total].increment(labels, 1) + end + + realtime = Benchmark.realtime do + yield + end + + @metrics[:sidekiq_jobs_completion_seconds].observe(labels, realtime) + rescue Exception # rubocop: disable Lint/RescueException + @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) + raise + ensure + @metrics[:sidekiq_running_jobs].increment(labels, -1) + end + + private + + def init_metrics + { + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete sidekiq job', buckets: SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), + sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :livesum) + } + end + + def create_labels(queue) + { + queue: queue + } + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/monitor.rb b/lib/gitlab/sidekiq_middleware/monitor.rb new file mode 100644 index 00000000000..53a6132edac --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/monitor.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class Monitor + def call(worker, job, queue) + Gitlab::SidekiqMonitor.instance.within_job(job['jid'], queue) do + yield + end + rescue Gitlab::SidekiqMonitor::CancelledError + # push job to DeadSet + payload = ::Sidekiq.dump_json(job) + ::Sidekiq::DeadSet.new.kill(payload, notify_failure: false) + + # ignore retries + raise ::Sidekiq::JobRetry::Skip + end + end + end +end diff --git a/lib/gitlab/sidekiq_monitor.rb b/lib/gitlab/sidekiq_monitor.rb new file mode 100644 index 00000000000..9842f1f53f7 --- /dev/null +++ b/lib/gitlab/sidekiq_monitor.rb @@ -0,0 +1,182 @@ +# frozen_string_literal: true + +module Gitlab + class SidekiqMonitor < Daemon + include ::Gitlab::Utils::StrongMemoize + + NOTIFICATION_CHANNEL = 'sidekiq:cancel:notifications'.freeze + CANCEL_DEADLINE = 24.hours.seconds + RECONNECT_TIME = 3.seconds + + # We use exception derived from `Exception` + # to consider this as an very low-level exception + # that should not be caught by application + CancelledError = Class.new(Exception) # rubocop:disable Lint/InheritException + + attr_reader :jobs_thread + attr_reader :jobs_mutex + + def initialize + super + + @jobs_thread = {} + @jobs_mutex = Mutex.new + end + + def within_job(jid, queue) + jobs_mutex.synchronize do + jobs_thread[jid] = Thread.current + end + + if cancelled?(jid) + Sidekiq.logger.warn( + class: self.class.to_s, + action: 'run', + queue: queue, + jid: jid, + canceled: true + ) + raise CancelledError + end + + yield + ensure + jobs_mutex.synchronize do + jobs_thread.delete(jid) + end + end + + def self.cancel_job(jid) + payload = { + action: 'cancel', + jid: jid + }.to_json + + ::Gitlab::Redis::SharedState.with do |redis| + redis.setex(cancel_job_key(jid), CANCEL_DEADLINE, 1) + redis.publish(NOTIFICATION_CHANNEL, payload) + end + end + + private + + def start_working + Sidekiq.logger.info( + class: self.class.to_s, + action: 'start', + message: 'Starting Monitor Daemon' + ) + + while enabled? + process_messages + sleep(RECONNECT_TIME) + end + + ensure + Sidekiq.logger.warn( + class: self.class.to_s, + action: 'stop', + message: 'Stopping Monitor Daemon' + ) + end + + def stop_working + thread.raise(Interrupt) if thread.alive? + end + + def process_messages + ::Gitlab::Redis::SharedState.with do |redis| + redis.subscribe(NOTIFICATION_CHANNEL) do |on| + on.message do |channel, message| + process_message(message) + end + end + end + rescue Exception => e # rubocop:disable Lint/RescueException + Sidekiq.logger.warn( + class: self.class.to_s, + action: 'exception', + message: e.message + ) + + # we re-raise system exceptions + raise e unless e.is_a?(StandardError) + end + + def process_message(message) + Sidekiq.logger.info( + class: self.class.to_s, + channel: NOTIFICATION_CHANNEL, + message: 'Received payload on channel', + payload: message + ) + + message = safe_parse(message) + return unless message + + case message['action'] + when 'cancel' + process_job_cancel(message['jid']) + else + # unknown message + end + end + + def safe_parse(message) + JSON.parse(message) + rescue JSON::ParserError + end + + def process_job_cancel(jid) + return unless jid + + # try to find thread without lock + return unless find_thread_unsafe(jid) + + Thread.new do + # try to find a thread, but with guaranteed + # that handle for thread corresponds to actually + # running job + find_thread_with_lock(jid) do |thread| + Sidekiq.logger.warn( + class: self.class.to_s, + action: 'cancel', + message: 'Canceling thread with CancelledError', + jid: jid, + thread_id: thread.object_id + ) + + thread&.raise(CancelledError) + end + end + end + + # This method needs to be thread-safe + # This is why it passes thread in block, + # to ensure that we do process this thread + def find_thread_unsafe(jid) + jobs_thread[jid] + end + + def find_thread_with_lock(jid) + # don't try to lock if we cannot find the thread + return unless find_thread_unsafe(jid) + + jobs_mutex.synchronize do + find_thread_unsafe(jid).tap do |thread| + yield(thread) if thread + end + end + end + + def cancelled?(jid) + ::Gitlab::Redis::SharedState.with do |redis| + redis.exists(self.class.cancel_job_key(jid)) + end + end + + def self.cancel_job_key(jid) + "sidekiq:cancel:#{jid}" + end + end +end diff --git a/lib/gitlab/slug/environment.rb b/lib/gitlab/slug/environment.rb new file mode 100644 index 00000000000..1b87d3bb626 --- /dev/null +++ b/lib/gitlab/slug/environment.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# An environment name is not necessarily suitable for use in URLs, DNS +# or other third-party contexts, so provide a slugified version. A slug has +# the following properties: +# * contains only lowercase letters (a-z), numbers (0-9), and '-' +# * begins with a letter +# * has a maximum length of 24 bytes (OpenShift limitation) +# * cannot end with `-` +module Gitlab + module Slug + class Environment + attr_reader :name + + def initialize(name) + @name = name + end + + def generate + # Lowercase letters and numbers only + slugified = name.to_s.downcase.gsub(/[^a-z0-9]/, '-') + + # Must start with a letter + slugified = 'env-' + slugified unless slugified.match?(/^[a-z]/) + + # Repeated dashes are invalid (OpenShift limitation) + slugified.squeeze!('-') + + slugified = + if slugified.size > 24 || slugified != name + # Maximum length: 24 characters (OpenShift limitation) + shorten_and_add_suffix(slugified) + else + # Cannot end with a dash (Kubernetes label limitation) + slugified.chomp('-') + end + + slugified + end + + private + + def shorten_and_add_suffix(slug) + slug = slug[0..16] + slug << '-' unless slug.ends_with?('-') + slug << suffix + end + + # Slugifying a name may remove the uniqueness guarantee afforded by it being + # based on name (which must be unique). To compensate, we add a predictable + # 6-byte suffix in those circumstances. This is not *guaranteed* uniqueness, + # but the chance of collisions is vanishingly small + def suffix + Digest::SHA2.hexdigest(name.to_s).to_i(16).to_s(36).last(6) + end + end + end +end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index e360b552f89..ac3b219e0c7 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -22,6 +22,17 @@ module Gitlab end end + def formatted_count(scope) + case scope + when 'snippet_titles' + snippet_titles_count.to_s + when 'snippet_blobs' + snippet_blobs_count.to_s + else + super + end + end + def snippet_titles_count @snippet_titles_count ||= snippet_titles.count end diff --git a/lib/gitlab/snowplow_tracker.rb b/lib/gitlab/snowplow_tracker.rb new file mode 100644 index 00000000000..9f12513e09e --- /dev/null +++ b/lib/gitlab/snowplow_tracker.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'snowplow-tracker' + +module Gitlab + module SnowplowTracker + NAMESPACE = 'cf' + + class << self + def track_event(category, action, label: nil, property: nil, value: nil, context: nil) + tracker&.track_struct_event(category, action, label, property, value, context, Time.now.to_i) + end + + private + + def tracker + return unless enabled? + + @tracker ||= ::SnowplowTracker::Tracker.new(emitter, subject, NAMESPACE, Gitlab::CurrentSettings.snowplow_site_id) + end + + def subject + ::SnowplowTracker::Subject.new + end + + def emitter + ::SnowplowTracker::Emitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname) + end + + def enabled? + Gitlab::CurrentSettings.snowplow_enabled? + end + end + end +end diff --git a/lib/gitlab/submodule_links.rb b/lib/gitlab/submodule_links.rb new file mode 100644 index 00000000000..18fd604a3b0 --- /dev/null +++ b/lib/gitlab/submodule_links.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + class SubmoduleLinks + include Gitlab::Utils::StrongMemoize + + def initialize(repository) + @repository = repository + end + + def for(submodule, sha) + submodule_url = submodule_url_for(sha, submodule.path) + SubmoduleHelper.submodule_links_for_url(submodule.id, submodule_url, repository) + end + + private + + attr_reader :repository + + def submodule_urls_for(sha) + strong_memoize(:"submodule_urls_for_#{sha}") do + repository.submodule_urls_for(sha) + end + end + + def submodule_url_for(sha, path) + urls = submodule_urls_for(sha) + urls && urls[path] + end + end +end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 9a8df719827..9c35d200dcb 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -18,7 +18,6 @@ module Gitlab # enforce_sanitization - Raises error if URL includes any HTML/CSS/JS tags and argument is true. # # Returns an array with [<uri>, <original-hostname>]. - # rubocop:disable Metrics/CyclomaticComplexity # rubocop:disable Metrics/ParameterLists def validate!( url, @@ -30,7 +29,6 @@ module Gitlab enforce_user: false, enforce_sanitization: false, dns_rebind_protection: true) - # rubocop:enable Metrics/CyclomaticComplexity # rubocop:enable Metrics/ParameterLists return [nil, nil] if url.nil? @@ -38,36 +36,34 @@ module Gitlab # Param url can be a string, URI or Addressable::URI uri = parse_url(url) - validate_html_tags!(uri) if enforce_sanitization + validate_uri( + uri: uri, + schemes: schemes, + ports: ports, + enforce_sanitization: enforce_sanitization, + enforce_user: enforce_user, + ascii_only: ascii_only + ) + normalized_hostname = uri.normalized_host hostname = uri.hostname port = get_port(uri) - unless internal?(uri) - validate_scheme!(uri.scheme, schemes) - validate_port!(port, ports) if ports.any? - validate_user!(uri.user) if enforce_user - validate_hostname!(hostname) - validate_unicode_restriction!(uri) if ascii_only - end - - begin - addrs_info = Addrinfo.getaddrinfo(hostname, port, nil, :STREAM).map do |addr| - addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr - end - rescue SocketError - return [uri, nil] - end + address_info = get_address_info(hostname, port) + return [uri, nil] unless address_info - protected_uri_with_hostname = enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) + ip_address = ip_address(address_info) + protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, hostname, dns_rebind_protection) # Allow url from the GitLab instance itself but only for the configured hostname and ports return protected_uri_with_hostname if internal?(uri) - validate_localhost!(addrs_info) unless allow_localhost - validate_loopback!(addrs_info) unless allow_localhost - validate_local_network!(addrs_info) unless allow_local_network - validate_link_local!(addrs_info) unless allow_local_network + validate_local_request( + normalized_hostname: normalized_hostname, + address_info: address_info, + allow_localhost: allow_localhost, + allow_local_network: allow_local_network + ) protected_uri_with_hostname end @@ -90,10 +86,7 @@ module Gitlab # # The original hostname is used to validate the SSL, given in that scenario # we'll be making the request to the IP address, instead of using the hostname. - def enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) - address = addrs_info.first - ip_address = address&.ip_address - + def enforce_uri_hostname(ip_address, uri, hostname, dns_rebind_protection) return [uri, nil] unless dns_rebind_protection && ip_address && ip_address != hostname uri = uri.dup @@ -101,11 +94,67 @@ module Gitlab [uri, hostname] end + def ip_address(address_info) + address_info.first&.ip_address + end + + def validate_uri(uri:, schemes:, ports:, enforce_sanitization:, enforce_user:, ascii_only:) + validate_html_tags(uri) if enforce_sanitization + + return if internal?(uri) + + validate_scheme(uri.scheme, schemes) + validate_port(get_port(uri), ports) if ports.any? + validate_user(uri.user) if enforce_user + validate_hostname(uri.hostname) + validate_unicode_restriction(uri) if ascii_only + end + + def get_address_info(hostname, port) + Addrinfo.getaddrinfo(hostname, port, nil, :STREAM).map do |addr| + addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr + end + rescue SocketError + # In the test suite we use a lot of mocked urls that are either invalid or + # don't exist. In order to avoid modifying a ton of tests and factories + # we allow invalid urls unless the environment variable RSPEC_ALLOW_INVALID_URLS + # is not true + return if Rails.env.test? && ENV['RSPEC_ALLOW_INVALID_URLS'] == 'true' + + # If the addr can't be resolved or the url is invalid (i.e http://1.1.1.1.1) + # we block the url + raise BlockedUrlError, "Host cannot be resolved or invalid" + end + + def validate_local_request( + normalized_hostname:, + address_info:, + allow_localhost:, + allow_local_network:) + return if allow_local_network && allow_localhost + + ip_whitelist, domain_whitelist = + Gitlab::CurrentSettings.outbound_local_requests_whitelist_arrays + + return if local_domain_whitelisted?(domain_whitelist, normalized_hostname) || + local_ip_whitelisted?(ip_whitelist, ip_address(address_info)) + + unless allow_localhost + validate_localhost(address_info) + validate_loopback(address_info) + end + + unless allow_local_network + validate_local_network(address_info) + validate_link_local(address_info) + end + end + def get_port(uri) uri.port || uri.default_port end - def validate_html_tags!(uri) + def validate_html_tags(uri) uri_str = uri.to_s sanitized_uri = ActionController::Base.helpers.sanitize(uri_str, tags: []) if sanitized_uri != uri_str @@ -125,7 +174,7 @@ module Gitlab CGI.unescape(url.to_s) =~ /\n|\r/ end - def validate_port!(port, ports) + def validate_port(port, ports) return if port.blank? # Only ports under 1024 are restricted return if port >= 1024 @@ -134,20 +183,20 @@ module Gitlab raise BlockedUrlError, "Only allowed ports are #{ports.join(', ')}, and any over 1024" end - def validate_scheme!(scheme, schemes) + def validate_scheme(scheme, schemes) if scheme.blank? || (schemes.any? && !schemes.include?(scheme)) raise BlockedUrlError, "Only allowed schemes are #{schemes.join(', ')}" end end - def validate_user!(value) + def validate_user(value) return if value.blank? return if value =~ /\A\p{Alnum}/ raise BlockedUrlError, "Username needs to start with an alphanumeric character" end - def validate_hostname!(value) + def validate_hostname(value) return if value.blank? return if IPAddress.valid?(value) return if value =~ /\A\p{Alnum}/ @@ -155,13 +204,13 @@ module Gitlab raise BlockedUrlError, "Hostname or IP address invalid" end - def validate_unicode_restriction!(uri) + def validate_unicode_restriction(uri) return if uri.to_s.ascii_only? raise BlockedUrlError, "URI must be ascii only #{uri.to_s.dump}" end - def validate_localhost!(addrs_info) + def validate_localhost(addrs_info) local_ips = ["::", "0.0.0.0"] local_ips.concat(Socket.ip_address_list.map(&:ip_address)) @@ -170,19 +219,19 @@ module Gitlab raise BlockedUrlError, "Requests to localhost are not allowed" end - def validate_loopback!(addrs_info) + def validate_loopback(addrs_info) return unless addrs_info.any? { |addr| addr.ipv4_loopback? || addr.ipv6_loopback? } raise BlockedUrlError, "Requests to loopback addresses are not allowed" end - def validate_local_network!(addrs_info) + def validate_local_network(addrs_info) return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? || addr.ipv6_unique_local? } raise BlockedUrlError, "Requests to the local network are not allowed" end - def validate_link_local!(addrs_info) + def validate_link_local(addrs_info) netmask = IPAddr.new('169.254.0.0/16') return unless addrs_info.any? { |addr| addr.ipv6_linklocal? || netmask.include?(addr.ip_address) } @@ -205,6 +254,16 @@ module Gitlab (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port) end + def local_ip_whitelisted?(ip_whitelist, ip_string) + ip_obj = Gitlab::Utils.string_to_ip_object(ip_string) + + ip_whitelist.any? { |ip| ip.include?(ip_obj) } + end + + def local_domain_whitelisted?(domain_whitelist, domain_string) + domain_whitelist.include?(domain_string) + end + def config Gitlab.config end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 0180fe7fa71..a93301cb4ce 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -6,7 +6,9 @@ module Gitlab class << self def data(force_refresh: false) - Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } + Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do + uncached_data + end end def uncached_data @@ -98,9 +100,7 @@ module Gitlab .merge(services_usage) .merge(approximate_counts) }.tap do |data| - if Feature.enabled?(:group_overview_security_dashboard) - data[:counts][:user_preferences] = user_preferences_usage - end + data[:counts][:user_preferences] = user_preferences_usage end end # rubocop: enable CodeReuse/ActiveRecord @@ -128,10 +128,23 @@ module Gitlab } end + # @return [Hash<Symbol, Integer>] def usage_counters - { - web_ide_commits: Gitlab::WebIdeCommitsCounter.total_count - } + usage_data_counters.map(&:totals).reduce({}) { |a, b| a.merge(b) } + end + + # @return [Array<#totals>] An array of objects that respond to `#totals` + def usage_data_counters + [ + Gitlab::UsageDataCounters::WikiPageCounter, + Gitlab::UsageDataCounters::WebIdeCounter, + Gitlab::UsageDataCounters::NoteCounter, + Gitlab::UsageDataCounters::SnippetCounter, + Gitlab::UsageDataCounters::SearchCounter, + Gitlab::UsageDataCounters::CycleAnalyticsCounter, + Gitlab::UsageDataCounters::SourceCodeCounter, + Gitlab::UsageDataCounters::MergeRequestCounter + ] end def components_usage_data @@ -176,8 +189,8 @@ module Gitlab {} # augmented in EE end - def count(relation, fallback: -1) - relation.count + def count(relation, count_by: nil, fallback: -1) + count_by ? relation.count(count_by) : relation.count rescue ActiveRecord::StatementInvalid fallback end diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb new file mode 100644 index 00000000000..2b52571c3cc --- /dev/null +++ b/lib/gitlab/usage_data_counters/base_counter.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class BaseCounter + extend RedisCounter + + UnknownEvent = Class.new(StandardError) + + class << self + def redis_key(event) + Gitlab::Sentry.track_exception(UnknownEvent, extra: { event: event }) unless known_events.include?(event.to_s) + + "USAGE_#{prefix}_#{event}".upcase + end + + def count(event) + increment(redis_key event) + end + + def read(event) + total_count(redis_key event) + end + + def totals + known_events.map { |e| ["#{prefix}_#{e}".to_sym, read(e)] }.to_h + end + + private + + def known_events + self::KNOWN_EVENTS + end + + def prefix + self::PREFIX + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/cycle_analytics_counter.rb b/lib/gitlab/usage_data_counters/cycle_analytics_counter.rb new file mode 100644 index 00000000000..1ff4296ef65 --- /dev/null +++ b/lib/gitlab/usage_data_counters/cycle_analytics_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class CycleAnalyticsCounter < BaseCounter + KNOWN_EVENTS = %w[views].freeze + PREFIX = 'cycle_analytics' + end +end diff --git a/lib/gitlab/usage_data_counters/merge_request_counter.rb b/lib/gitlab/usage_data_counters/merge_request_counter.rb new file mode 100644 index 00000000000..e786e595f77 --- /dev/null +++ b/lib/gitlab/usage_data_counters/merge_request_counter.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class MergeRequestCounter < BaseCounter + KNOWN_EVENTS = %w[create].freeze + PREFIX = 'merge_request' + end + end +end diff --git a/lib/gitlab/usage_data_counters/note_counter.rb b/lib/gitlab/usage_data_counters/note_counter.rb new file mode 100644 index 00000000000..672450ec82b --- /dev/null +++ b/lib/gitlab/usage_data_counters/note_counter.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class NoteCounter < BaseCounter + KNOWN_EVENTS = %w[create].freeze + PREFIX = 'note' + COUNTABLE_TYPES = %w[Snippet Commit MergeRequest].freeze + + class << self + def redis_key(event, noteable_type) + "#{super(event)}_#{noteable_type}".upcase + end + + def count(event, noteable_type) + return unless countable?(noteable_type) + + increment(redis_key(event, noteable_type)) + end + + def read(event, noteable_type) + return 0 unless countable?(noteable_type) + + total_count(redis_key(event, noteable_type)) + end + + def totals + COUNTABLE_TYPES.map do |countable_type| + [:"#{countable_type.underscore}_comment", read(:create, countable_type)] + end.to_h + end + + private + + def countable?(noteable_type) + COUNTABLE_TYPES.include?(noteable_type.to_s) + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/redis_counter.rb b/lib/gitlab/usage_data_counters/redis_counter.rb new file mode 100644 index 00000000000..75d5a75e3a4 --- /dev/null +++ b/lib/gitlab/usage_data_counters/redis_counter.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module RedisCounter + def increment(redis_counter_key) + return unless Gitlab::CurrentSettings.usage_ping_enabled + + Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } + end + + def total_count(redis_counter_key) + Gitlab::Redis::SharedState.with { |redis| redis.get(redis_counter_key).to_i } + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/search_counter.rb b/lib/gitlab/usage_data_counters/search_counter.rb new file mode 100644 index 00000000000..5f0735347e1 --- /dev/null +++ b/lib/gitlab/usage_data_counters/search_counter.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class SearchCounter + extend RedisCounter + + NAVBAR_SEARCHES_COUNT_KEY = 'NAVBAR_SEARCHES_COUNT' + + class << self + def increment_navbar_searches_count + increment(NAVBAR_SEARCHES_COUNT_KEY) + end + + def total_navbar_searches_count + total_count(NAVBAR_SEARCHES_COUNT_KEY) + end + + def totals + { + navbar_searches: total_navbar_searches_count + } + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/snippet_counter.rb b/lib/gitlab/usage_data_counters/snippet_counter.rb new file mode 100644 index 00000000000..e4d234ce4d9 --- /dev/null +++ b/lib/gitlab/usage_data_counters/snippet_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class SnippetCounter < BaseCounter + KNOWN_EVENTS = %w[create update].freeze + PREFIX = 'snippet' + end +end diff --git a/lib/gitlab/usage_data_counters/source_code_counter.rb b/lib/gitlab/usage_data_counters/source_code_counter.rb new file mode 100644 index 00000000000..8a1771a7bd1 --- /dev/null +++ b/lib/gitlab/usage_data_counters/source_code_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class SourceCodeCounter < BaseCounter + KNOWN_EVENTS = %w[pushes].freeze + PREFIX = 'source_code' + end +end diff --git a/lib/gitlab/usage_data_counters/web_ide_counter.rb b/lib/gitlab/usage_data_counters/web_ide_counter.rb new file mode 100644 index 00000000000..0718c1dd761 --- /dev/null +++ b/lib/gitlab/usage_data_counters/web_ide_counter.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class WebIdeCounter + extend RedisCounter + + COMMITS_COUNT_KEY = 'WEB_IDE_COMMITS_COUNT' + MERGE_REQUEST_COUNT_KEY = 'WEB_IDE_MERGE_REQUESTS_COUNT' + VIEWS_COUNT_KEY = 'WEB_IDE_VIEWS_COUNT' + + class << self + def increment_commits_count + increment(COMMITS_COUNT_KEY) + end + + def total_commits_count + total_count(COMMITS_COUNT_KEY) + end + + def increment_merge_requests_count + increment(MERGE_REQUEST_COUNT_KEY) + end + + def total_merge_requests_count + total_count(MERGE_REQUEST_COUNT_KEY) + end + + def increment_views_count + increment(VIEWS_COUNT_KEY) + end + + def total_views_count + total_count(VIEWS_COUNT_KEY) + end + + def totals + { + web_ide_commits: total_commits_count, + web_ide_views: total_views_count, + web_ide_merge_requests: total_merge_requests_count + } + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/wiki_page_counter.rb b/lib/gitlab/usage_data_counters/wiki_page_counter.rb new file mode 100644 index 00000000000..9cfe0be5bab --- /dev/null +++ b/lib/gitlab/usage_data_counters/wiki_page_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class WikiPageCounter < BaseCounter + KNOWN_EVENTS = %w[create update delete].freeze + PREFIX = 'wiki_pages' + end +end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 16ec8a8bb28..c66ce0434a4 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -22,7 +22,7 @@ module Gitlab end def force_utf8(str) - str.force_encoding(Encoding::UTF_8) + str.dup.force_encoding(Encoding::UTF_8) end def ensure_utf8_size(str, bytes:) @@ -131,5 +131,12 @@ module Gitlab data end end + + def string_to_ip_object(str) + return unless str + + IPAddr.new(str) + rescue IPAddr::InvalidAddressError + end end end diff --git a/lib/gitlab/utils/sanitize_node_link.rb b/lib/gitlab/utils/sanitize_node_link.rb new file mode 100644 index 00000000000..620d71a7814 --- /dev/null +++ b/lib/gitlab/utils/sanitize_node_link.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require_dependency 'gitlab/utils' + +module Gitlab + module Utils + module SanitizeNodeLink + UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + ATTRS_TO_SANITIZE = %w(href src data-src).freeze + + def remove_unsafe_links(env, remove_invalid_links: true) + node = env[:node] + + sanitize_node(node: node, remove_invalid_links: remove_invalid_links) + + # HTML entities such as <video></video> have scannable attrs in + # children elements, which also need to be sanitized. + # + node.children.each do |child_node| + sanitize_node(node: child_node, remove_invalid_links: remove_invalid_links) + end + end + + # Remove all invalid scheme characters before checking against the + # list of unsafe protocols. + # + # See https://tools.ietf.org/html/rfc3986#section-3.1 + # + def safe_protocol?(scheme) + return false unless scheme + + scheme = scheme + .strip + .downcase + .gsub(/[^A-Za-z\+\.\-]+/, '') + + UNSAFE_PROTOCOLS.none?(scheme) + end + + private + + def sanitize_node(node:, remove_invalid_links: true) + ATTRS_TO_SANITIZE.each do |attr| + next unless node.has_attribute?(attr) + + begin + node[attr] = node[attr].strip + uri = Addressable::URI.parse(node[attr]) + + next unless uri.scheme + next if safe_protocol?(uri.scheme) + + node.remove_attribute(attr) + rescue Addressable::URI::InvalidURIError + node.remove_attribute(attr) if remove_invalid_links + end + end + end + end + end +end diff --git a/lib/gitlab/web_ide_commits_counter.rb b/lib/gitlab/web_ide_commits_counter.rb deleted file mode 100644 index 1cd9b5295b9..00000000000 --- a/lib/gitlab/web_ide_commits_counter.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module WebIdeCommitsCounter - WEB_IDE_COMMITS_KEY = "WEB_IDE_COMMITS_COUNT".freeze - - class << self - def increment - Gitlab::Redis::SharedState.with { |redis| redis.incr(WEB_IDE_COMMITS_KEY) } - end - - def total_count - Gitlab::Redis::SharedState.with { |redis| redis.get(WEB_IDE_COMMITS_KEY).to_i } - end - end - end -end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 46a7b5b982a..3b77fe838ae 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -221,7 +221,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis::Queues.with do |redis| + Gitlab::Redis::SharedState.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") diff --git a/lib/gitlab/zoom_link_extractor.rb b/lib/gitlab/zoom_link_extractor.rb new file mode 100644 index 00000000000..7ac14eb2d4f --- /dev/null +++ b/lib/gitlab/zoom_link_extractor.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# Detect links matching the following formats: +# Zoom Start links: https://zoom.us/s/<meeting-id> +# Zoom Join links: https://zoom.us/j/<meeting-id> +# Personal Zoom links: https://zoom.us/my/<meeting-id> +# Vanity Zoom links: https://gitlab.zoom.us/j/<meeting-id> (also /s and /my) + +module Gitlab + class ZoomLinkExtractor + ZOOM_REGEXP = %r{https://(?:[\w-]+\.)?zoom\.us/(?:s|j|my)/\S+}.freeze + + def initialize(text) + @text = text.to_s + end + + def links + @text.scan(ZOOM_REGEXP) + end + + def match? + ZOOM_REGEXP.match?(@text) + end + end +end diff --git a/lib/gt_one_coercion.rb b/lib/gt_one_coercion.rb deleted file mode 100644 index 99be51bc8c6..00000000000 --- a/lib/gt_one_coercion.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -class GtOneCoercion < Virtus::Attribute - def coerce(value) - [1, value.to_i].max - end -end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index 722e3e04d1c..eea7daa3d8e 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -41,7 +41,7 @@ module Mattermost begin yield self rescue Errno::ECONNREFUSED => e - Rails.logger.error(e.message + "\n" + e.backtrace.join("\n")) + Rails.logger.error(e.message + "\n" + e.backtrace.join("\n")) # rubocop:disable Gitlab/RailsLogger raise Mattermost::NoSessionError ensure destroy diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb index c7dec09ba6b..a7dcd322e27 100644 --- a/lib/microsoft_teams/notifier.rb +++ b/lib/microsoft_teams/notifier.rb @@ -20,7 +20,7 @@ module MicrosoftTeams result = true if response rescue Gitlab::HTTP::Error, StandardError => error - Rails.logger.info("#{self.class.name}: Error while connecting to #{@webhook}: #{error.message}") + Rails.logger.info("#{self.class.name}: Error while connecting to #{@webhook}: #{error.message}") # rubocop:disable Gitlab/RailsLogger end result diff --git a/lib/mysql_zero_date.rb b/lib/mysql_zero_date.rb deleted file mode 100644 index f36610abf8f..00000000000 --- a/lib/mysql_zero_date.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -# Disable NO_ZERO_DATE mode for mysql in rails 5. -# We use zero date as a default value -# (config/initializers/active_record_mysql_timestamp.rb), in -# Rails 5 using zero date fails by default (https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/75450216) -# and NO_ZERO_DATE has to be explicitly disabled. Disabling strict mode -# is not sufficient. - -require 'active_record/connection_adapters/abstract_mysql_adapter' - -module MysqlZeroDate - def configure_connection - super - - @connection.query "SET @@SESSION.sql_mode = REPLACE(@@SESSION.sql_mode, 'NO_ZERO_DATE', '');" # rubocop:disable Gitlab/ModuleWithInstanceVariables - end -end - -ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlZeroDate) diff --git a/lib/peek/rblineprof/custom_controller_helpers.rb b/lib/peek/rblineprof/custom_controller_helpers.rb deleted file mode 100644 index 581cc6a37b4..00000000000 --- a/lib/peek/rblineprof/custom_controller_helpers.rb +++ /dev/null @@ -1,124 +0,0 @@ -# frozen_string_literal: true - -module Peek - module Rblineprof - module CustomControllerHelpers - extend ActiveSupport::Concern - - # This will become useless once https://github.com/peek/peek-rblineprof/pull/5 - # is merged - def pygmentize(file_name, code, lexer = nil) - if lexer.present? - Gitlab::Highlight.highlight(file_name, code) - else - "<pre>#{Rack::Utils.escape_html(code)}</pre>" - end - end - - # rubocop:disable all - def inject_rblineprof - ret = nil - profile = lineprof(rblineprof_profiler_regex) do - ret = yield - end - - if response.content_type =~ %r|text/html| - sort = params[:lineprofiler_sort] - mode = params[:lineprofiler_mode] || 'cpu' - min = (params[:lineprofiler_min] || 5).to_i * 1000 - summary = params[:lineprofiler_summary] - - # Sort each file by the longest calculated time - per_file = profile.map do |file, lines| - total, child, excl, total_cpu, child_cpu, excl_cpu = lines[0] - - wall = summary == 'exclusive' ? excl : total - cpu = summary == 'exclusive' ? excl_cpu : total_cpu - idle = summary == 'exclusive' ? (excl - excl_cpu) : (total - total_cpu) - - [ - file, lines, - wall, cpu, idle, - sort == 'idle' ? idle : sort == 'cpu' ? cpu : wall - ] - end.sort_by{ |a,b,c,d,e,f| -f } - - output = ["<div class='modal-dialog modal-xl'><div class='modal-content'>"] - output << "<div class='modal-header'>" - output << "<h4>Line profiling: #{human_description(params[:lineprofiler])}</h4>" - output << "<button class='close' type='button' data-dismiss='modal' aria-label='close'><span aria-hidden='true'>×</span></button>" - output << "</div>" - output << "<div class='modal-body'>" - - per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort| - output << "<div class='peek-rblineprof-file'><div class='heading'>" - - show_src = file_sort > min - tmpl = show_src ? "<a href='#' class='js-lineprof-file'>%s</a>" : "%s" - - if mode == 'cpu' - output << sprintf("<span class='duration'>% 8.1fms + % 8.1fms</span> #{tmpl}", file_cpu / 1000.0, file_idle / 1000.0, file_name.sub(Rails.root.to_s + '/', '')) - else - output << sprintf("<span class='duration'>% 8.1fms</span> #{tmpl}", file_wall/1000.0, file_name.sub(Rails.root.to_s + '/', '')) - end - - output << "</div>" # .heading - - next unless show_src - - output << "<div class='data'>" - code = [] - times = [] - File.readlines(file_name).each_with_index do |line, i| - code << line - wall, cpu, calls = lines[i + 1] - - if calls && calls > 0 - if mode == 'cpu' - idle = wall - cpu - times << sprintf("% 8.1fms + % 8.1fms (% 5d)", cpu / 1000.0, idle / 1000.0, calls) - else - times << sprintf("% 8.1fms (% 5d)", wall / 1000.0, calls) - end - else - times << ' ' - end - end - output << "<pre class='duration'>#{times.join("\n")}</pre>" - # The following line was changed from - # https://github.com/peek/peek-rblineprof/blob/8d3b7a283a27de2f40abda45974516693d882258/lib/peek/rblineprof/controller_helpers.rb#L125 - # This will become useless once https://github.com/peek/peek-rblineprof/pull/16 - # is merged and is implemented. - output << "<pre class='code highlight white'>#{pygmentize(file_name, code.join, 'ruby')}</pre>" - output << "</div></div>" # .data then .peek-rblineprof-file - end - - output << "</div></div></div>" - - response.body += "<div class='modal' id='modal-peek-line-profile' tabindex=-1>#{output.join}</div>".html_safe - end - - ret - end - - private - - def human_description(lineprofiler_param) - case lineprofiler_param - when 'app' - 'app/ & lib/' - when 'views' - 'app/view/' - when 'gems' - 'vendor/gems' - when 'all' - 'everything in Rails.root' - when 'stdlib' - 'everything in the Ruby standard library' - else - 'app/, config/, lib/, vendor/ & plugin/' - end - end - end - end -end diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb new file mode 100644 index 00000000000..2d78818630d --- /dev/null +++ b/lib/peek/views/active_record.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Peek + module Views + class ActiveRecord < DetailedView + private + + def setup_subscribers + super + + subscribe('sql.active_record') do |_, start, finish, _, data| + if Gitlab::SafeRequestStore.store[:peek_enabled] + unless data[:cached] + detail_store << { + duration: finish - start, + sql: data[:sql].strip, + backtrace: Gitlab::Profiler.clean_backtrace(caller) + } + end + end + end + end + end + end +end diff --git a/lib/peek/views/detailed_view.rb b/lib/peek/views/detailed_view.rb new file mode 100644 index 00000000000..f4ca1cb5075 --- /dev/null +++ b/lib/peek/views/detailed_view.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Peek + module Views + class DetailedView < View + def results + { + duration: formatted_duration, + calls: calls, + details: details + } + end + + def detail_store + ::Gitlab::SafeRequestStore["#{key}_call_details"] ||= [] + end + + private + + def duration + detail_store.map { |entry| entry[:duration] }.sum # rubocop:disable CodeReuse/ActiveRecord + end + + def calls + detail_store.count + end + + def call_details + detail_store + end + + def format_call_details(call) + call.merge(duration: (call[:duration] * 1000).round(3)) + end + + def details + call_details + .sort { |a, b| b[:duration] <=> a[:duration] } + .map(&method(:format_call_details)) + end + + def formatted_duration + ms = duration * 1000 + + if ms >= 1000 + "%.2fms" % ms + else + "%.0fms" % ms + end + end + end + end +end diff --git a/lib/peek/views/gitaly.rb b/lib/peek/views/gitaly.rb index 30f95a10024..6ad6ddfd89d 100644 --- a/lib/peek/views/gitaly.rb +++ b/lib/peek/views/gitaly.rb @@ -2,7 +2,9 @@ module Peek module Views - class Gitaly < View + class Gitaly < DetailedView + private + def duration ::Gitlab::GitalyClient.query_time end @@ -11,36 +13,14 @@ module Peek ::Gitlab::GitalyClient.get_request_count end - def results - { - duration: formatted_duration, - calls: calls, - details: details - } - end - - private - - def details + def call_details ::Gitlab::GitalyClient.list_call_details - .sort { |a, b| b[:duration] <=> a[:duration] } - .map(&method(:format_call_details)) end def format_call_details(call) pretty_request = call[:request]&.reject { |k, v| v.blank? }.to_h.pretty_inspect - call.merge(duration: (call[:duration] * 1000).round(3), - request: pretty_request || {}) - end - - def formatted_duration - ms = duration * 1000 - if ms >= 1000 - "%.2fms" % ms - else - "%.0fms" % ms - end + super.merge(request: pretty_request || {}) end def setup_subscribers diff --git a/lib/peek/views/redis.rb b/lib/peek/views/redis_detailed.rb index 73de8672fa4..f36f581d5e9 100644 --- a/lib/peek/views/redis.rb +++ b/lib/peek/views/redis_detailed.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'redis' -require 'peek-redis' module Gitlab module Peek @@ -17,16 +16,21 @@ module Gitlab private def add_call_details(duration, args) + return unless peek_enabled? # redis-rb passes an array (e.g. [:get, key]) return unless args.length == 1 detail_store << { cmd: args.first, duration: duration, - backtrace: Gitlab::Profiler.clean_backtrace(caller) + backtrace: ::Gitlab::Profiler.clean_backtrace(caller) } end + def peek_enabled? + Gitlab::SafeRequestStore.store[:peek_enabled] + end + def detail_store ::Gitlab::SafeRequestStore['redis_call_details'] ||= [] end @@ -36,26 +40,17 @@ end module Peek module Views - module RedisDetailed + class RedisDetailed < DetailedView REDACTED_MARKER = "<redacted>" - def results - super.merge(details: details) - end - - def details - detail_store - .sort { |a, b| b[:duration] <=> a[:duration] } - .map(&method(:format_call_details)) + def key + 'redis' end - def detail_store - ::Gitlab::SafeRequestStore['redis_call_details'] ||= [] - end + private def format_call_details(call) - call.merge(cmd: format_command(call[:cmd]), - duration: (call[:duration] * 1000).round(3)) + super.merge(cmd: format_command(call[:cmd])) end def format_command(cmd) @@ -76,11 +71,3 @@ end class Redis::Client prepend Gitlab::Peek::RedisInstrumented end - -module Peek - module Views - class Redis < View - prepend Peek::Views::RedisDetailed - end - end -end diff --git a/lib/peek/views/rugged.rb b/lib/peek/views/rugged.rb new file mode 100644 index 00000000000..18b3f422852 --- /dev/null +++ b/lib/peek/views/rugged.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Peek + module Views + class Rugged < DetailedView + def results + return {} unless calls > 0 + + super + end + + private + + def duration + ::Gitlab::RuggedInstrumentation.query_time + end + + def calls + ::Gitlab::RuggedInstrumentation.query_count + end + + def call_details + ::Gitlab::RuggedInstrumentation.list_call_details + end + + def format_call_details(call) + super.merge(args: format_args(call[:args])) + end + + def format_args(args) + args.map do |arg| + # ActiveSupport::JSON recursively calls as_json on all + # instance variables, and if that instance variable points to + # something that refers back to the same instance, we can wind + # up in an infinite loop. Currently this only seems to happen with + # Gitlab::Git::Repository and ::Repository. + if arg.instance_variables.present? + arg.to_s + else + arg + end + end + end + end + end +end diff --git a/lib/prometheus/cleanup_multiproc_dir_service.rb b/lib/prometheus/cleanup_multiproc_dir_service.rb new file mode 100644 index 00000000000..6418b4de166 --- /dev/null +++ b/lib/prometheus/cleanup_multiproc_dir_service.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Prometheus + class CleanupMultiprocDirService + include Gitlab::Utils::StrongMemoize + + def execute + FileUtils.rm_rf(old_metrics) if old_metrics + end + + private + + def old_metrics + strong_memoize(:old_metrics) do + Dir[File.join(multiprocess_files_dir, '*.db')] if multiprocess_files_dir + end + end + + def multiprocess_files_dir + ::Prometheus::Client.configuration.multiprocess_files_dir + end + end +end diff --git a/lib/prometheus/pid_provider.rb b/lib/prometheus/pid_provider.rb new file mode 100644 index 00000000000..e0f7e7e0a9e --- /dev/null +++ b/lib/prometheus/pid_provider.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Prometheus + module PidProvider + extend self + + def worker_id + if Sidekiq.server? + 'sidekiq' + elsif defined?(Unicorn::Worker) + unicorn_worker_id + elsif defined?(::Puma) + puma_worker_id + else + unknown_process_id + end + end + + private + + def unicorn_worker_id + if matches = process_name.match(/unicorn.*worker\[([0-9]+)\]/) + "unicorn_#{matches[1]}" + elsif process_name =~ /unicorn/ + "unicorn_master" + else + unknown_process_id + end + end + + def puma_worker_id + if matches = process_name.match(/puma.*cluster worker ([0-9]+):/) + "puma_#{matches[1]}" + elsif process_name =~ /puma/ + "puma_master" + else + unknown_process_id + end + end + + def unknown_process_id + "process_#{Process.pid}" + end + + def process_name + $0 + end + end +end diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb index 24d8eac200c..60d79b52680 100644 --- a/lib/quality/test_level.rb +++ b/lib/quality/test_level.rb @@ -14,6 +14,7 @@ module Quality finders frontend graphql + haml_lint helpers initializers javascripts diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb index 19cc0baa2d3..bf15130d17e 100644 --- a/lib/rspec_flaky/listener.rb +++ b/lib/rspec_flaky/listener.rb @@ -32,6 +32,7 @@ module RspecFlaky flaky_examples[current_example.uid] = flaky_example end + # rubocop:disable Gitlab/RailsLogger def dump_summary(_) RspecFlaky::Report.new(flaky_examples).write(RspecFlaky::Config.flaky_examples_report_path) # write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path) @@ -45,6 +46,7 @@ module RspecFlaky # write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path) end end + # rubocop:enable Gitlab/RailsLogger private diff --git a/lib/sentry/client.rb b/lib/sentry/client.rb index 4022e8ff946..07cca1c8d1e 100644 --- a/lib/sentry/client.rb +++ b/lib/sentry/client.rb @@ -67,7 +67,7 @@ module Sentry def handle_request_exceptions yield - rescue HTTParty::Error => e + rescue Gitlab::HTTP::Error => e Gitlab::Sentry.track_acceptable_exception(e) raise_error 'Error when connecting to Sentry' rescue Net::OpenTimeout diff --git a/lib/serializers/json.rb b/lib/serializers/json.rb index 93cb192087a..1ed5d5dc3f5 100644 --- a/lib/serializers/json.rb +++ b/lib/serializers/json.rb @@ -1,32 +1,16 @@ # frozen_string_literal: true module Serializers - # This serializer exports data as JSON, - # it is designed to be used with interwork compatibility between MySQL and PostgreSQL - # implementations, as used version of MySQL does not support native json type - # - # Secondly, the loader makes the resulting hash to have deep indifferent access + # Make the resulting hash have deep indifferent access class JSON class << self def dump(obj) - # MySQL stores data as text - # look at ./config/initializers/ar_mysql_jsonb_support.rb - if Gitlab::Database.mysql? - obj = ActiveSupport::JSON.encode(obj) - end - obj end def load(data) return if data.nil? - # On MySQL we store data as text - # look at ./config/initializers/ar_mysql_jsonb_support.rb - if Gitlab::Database.mysql? - data = ActiveSupport::JSON.decode(data) - end - Gitlab::Utils.deep_indifferent_access(data) end end diff --git a/lib/support/deploy/deploy.sh b/lib/support/deploy/deploy.sh index ab46c47d8f5..1f8c915140c 100755 --- a/lib/support/deploy/deploy.sh +++ b/lib/support/deploy/deploy.sh @@ -28,7 +28,7 @@ sudo -u git -H git pull origin master echo 'Deploy: Bundle and migrate' # change it to your needs -sudo -u git -H bundle --without aws development test mysql --deployment +sudo -u git -H bundle --without aws development test --deployment sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:assets:clean RAILS_ENV=production diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 32df74f104a..bccb94ff0bf 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -67,6 +67,13 @@ if ! cd "$app_root" ; then echo "Failed to cd into $app_root, exiting!"; exit 1 fi +# Select the web server to use +if [ -z "$EXPERIMENTAL_PUMA" ]; then + use_web_server="unicorn" +else + use_web_server="puma" +fi + ### Init Script functions @@ -256,7 +263,7 @@ start_gitlab() { check_stale_pids if [ "$web_status" != "0" ]; then - echo "Starting GitLab web server" + echo "Starting GitLab web server ($use_web_server)" fi if [ "$sidekiq_status" != "0" ]; then echo "Starting GitLab Sidekiq" @@ -281,7 +288,7 @@ start_gitlab() { # Remove old socket if it exists rm -f "$rails_socket" 2>/dev/null # Start the web server - RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web start + RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web start fi # If sidekiq is already running, don't start it again. @@ -343,7 +350,7 @@ stop_gitlab() { if [ "$web_status" = "0" ]; then echo "Shutting down GitLab web server" - RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web stop + RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web stop fi if [ "$sidekiq_status" = "0" ]; then echo "Shutting down GitLab Sidekiq" @@ -447,7 +454,7 @@ reload_gitlab(){ exit 1 fi printf "Reloading GitLab web server configuration... " - RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web reload + RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web reload echo "Done." echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..." diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb index 467711fb74e..08c8df9b044 100644 --- a/lib/system_check/app/git_version_check.rb +++ b/lib/system_check/app/git_version_check.rb @@ -7,7 +7,7 @@ module SystemCheck set_check_pass -> { "yes (#{self.current_version})" } def self.required_version - @required_version ||= Gitlab::VersionInfo.parse('2.21.0') + @required_version ||= Gitlab::VersionInfo.parse('2.22.0') end def self.current_version diff --git a/lib/tasks/frontend.rake b/lib/tasks/frontend.rake new file mode 100644 index 00000000000..1cac7520227 --- /dev/null +++ b/lib/tasks/frontend.rake @@ -0,0 +1,21 @@ +unless Rails.env.production? + namespace :frontend do + desc 'GitLab | Frontend | Generate fixtures for JavaScript tests' + RSpec::Core::RakeTask.new(:fixtures, [:pattern]) do |t, args| + args.with_defaults(pattern: '{spec,ee/spec}/frontend/fixtures/*.rb') + ENV['NO_KNAPSACK'] = 'true' + t.pattern = args[:pattern] + t.rspec_opts = '--format documentation' + end + + desc 'GitLab | Frontend | Run JavaScript tests' + task tests: ['yarn:check'] do + sh "yarn test" do |ok, res| + abort('rake frontend:tests failed') unless ok + end + end + end + + desc 'GitLab | Frontend | Shortcut for frontend:fixtures and frontend:tests' + task frontend: ['frontend:fixtures', 'frontend:tests'] +end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index a07ae3a418a..7a42e4e92a0 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -10,15 +10,9 @@ namespace :gitlab do rake:assets:precompile webpack:compile gitlab:assets:fix_urls - gitlab:assets:compile_vrt ].each(&Gitlab::TaskHelpers.method(:invoke_and_time_task)) end - desc 'GitLab | Assets | Compile visual review toolbar' - task :compile_vrt do - system 'yarn', 'webpack-vrt' - end - desc 'GitLab | Assets | Clean up old compiled frontend assets' task clean: ['rake:assets:clean'] diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 105ef417df3..4d854cd178d 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -127,6 +127,58 @@ namespace :gitlab do end end + namespace :sessions do + desc "GitLab | Cleanup | Sessions | Clean ActiveSession lookup keys" + task active_sessions_lookup_keys: :gitlab_environment do + session_key_pattern = "#{Gitlab::Redis::SharedState::USER_SESSIONS_LOOKUP_NAMESPACE}:*" + last_save_check = Time.at(0) + wait_time = 10.seconds + cursor = 0 + total_users_scanned = 0 + + Gitlab::Redis::SharedState.with do |redis| + begin + cursor, keys = redis.scan(cursor, match: session_key_pattern) + total_users_scanned += keys.count + + if last_save_check < Time.now - 1.second + while redis.info('persistence')['rdb_bgsave_in_progress'] == '1' + puts "BGSAVE in progress, waiting #{wait_time} seconds" + sleep(wait_time) + end + last_save_check = Time.now + end + + keys.each do |key| + user_id = key.split(':').last + + lookup_key_count = redis.scard(key) + + session_ids = ActiveSession.session_ids_for_user(user_id) + entries = ActiveSession.raw_active_session_entries(session_ids, user_id) + session_ids_and_entries = session_ids.zip(entries) + + inactive_session_ids = session_ids_and_entries.map do |session_id, session| + session_id if session.nil? + end.compact + + redis.pipelined do |conn| + inactive_session_ids.each do |session_id| + conn.srem(key, session_id) + end + end + + if inactive_session_ids + puts "deleted #{inactive_session_ids.count} out of #{lookup_key_count} lookup keys for User ##{user_id}" + end + end + end while cursor.to_i != 0 + + puts "--- All done! Total number of scanned users: #{total_users_scanned}" + end + end + end + def remove? ENV['REMOVE'] == 'true' end @@ -147,6 +199,7 @@ namespace :gitlab do ENV['NICENESS'].presence end + # rubocop:disable Gitlab/RailsLogger def logger return @logger if defined?(@logger) @@ -159,5 +212,6 @@ namespace :gitlab do Rails.logger end end + # rubocop:enable Gitlab/RailsLogger end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 4e7a8adbef6..1961f64659c 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -26,26 +26,19 @@ namespace :gitlab do task drop_tables: :environment do connection = ActiveRecord::Base.connection - # If MySQL, turn off foreign key checks - connection.execute('SET FOREIGN_KEY_CHECKS=0') if Gitlab::Database.mysql? - - # connection.tables is deprecated in MySQLAdapter, but in PostgreSQLAdapter - # data_sources returns both views and tables, so use #tables instead - tables = Gitlab::Database.mysql? ? connection.data_sources : connection.tables + # In PostgreSQLAdapter, data_sources returns both views and tables, so use + # #tables instead + tables = connection.tables # Removes the entry from the array tables.delete 'schema_migrations' # Truncate schema_migrations to ensure migrations re-run - connection.execute('TRUNCATE schema_migrations') if connection.data_source_exists? 'schema_migrations' + connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations' # Drop tables with cascade to avoid dependent table errors # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html - # MySQL: http://dev.mysql.com/doc/refman/5.7/en/drop-table.html # Add `IF EXISTS` because cascade could have already deleted a table. tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") } - - # If MySQL, re-enable foreign key checks - connection.execute('SET FOREIGN_KEY_CHECKS=1') if Gitlab::Database.mysql? end desc 'Configures the database by running migrate, or by loading the schema and seeding if needed' @@ -77,5 +70,13 @@ namespace :gitlab do Gitlab::DowntimeCheck.new.check_and_print(migrations) end + + desc 'Sets up EE specific database functionality' + + if Gitlab.ee? + task setup_ee: %w[geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate] + else + task :setup_ee + end end end diff --git a/lib/tasks/gitlab/features.rake b/lib/tasks/gitlab/features.rake index d88bcca0819..9cf568c07fe 100644 --- a/lib/tasks/gitlab/features.rake +++ b/lib/tasks/gitlab/features.rake @@ -10,14 +10,22 @@ namespace :gitlab do set_rugged_feature_flags(false) puts 'All Rugged feature flags were disabled.' end + + task unset_rugged: :environment do + set_rugged_feature_flags(nil) + puts 'All Rugged feature flags were unset.' + end end def set_rugged_feature_flags(status) Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.each do |flag| - if status - Feature.enable(flag) - else + case status + when nil Feature.get(flag).remove + when true + Feature.enable(flag) + when false + Feature.disable(flag) end end end diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake new file mode 100644 index 00000000000..c53d55ceea2 --- /dev/null +++ b/lib/tasks/gitlab/graphql.rake @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +namespace :gitlab do + OUTPUT_DIR = Rails.root.join("doc/api/graphql/reference").freeze + TEMPLATES_DIR = 'lib/gitlab/graphql/docs/templates/'.freeze + + namespace :graphql do + desc 'GitLab | Generate GraphQL docs' + task compile_docs: :environment do + renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options) + + renderer.render + + puts "Documentation compiled." + end + end +end + +def render_options + { + output_dir: OUTPUT_DIR, + template: Rails.root.join(TEMPLATES_DIR, 'default.md.haml') + } +end diff --git a/lib/tasks/gitlab/seed.rake b/lib/tasks/gitlab/seed.rake index 155ba979b36..d76e38b73b5 100644 --- a/lib/tasks/gitlab/seed.rake +++ b/lib/tasks/gitlab/seed.rake @@ -1,7 +1,9 @@ namespace :gitlab do namespace :seed do desc "GitLab | Seed | Seeds issues" - task :issues, [:project_full_path] => :environment do |t, args| + task :issues, [:project_full_path, :backfill_weeks, :average_issues_per_week] => :environment do |t, args| + args.with_defaults(backfill_weeks: 5, average_issues_per_week: 2) + projects = if args.project_full_path project = Project.find_by_full_path(args.project_full_path) @@ -26,7 +28,8 @@ namespace :gitlab do projects.each do |project| puts "\nSeeding issues for the '#{project.full_path}' project" seeder = Quality::Seeders::Issues.new(project: project) - issues_created = seeder.seed(backfill_weeks: 5, average_issues_per_week: 2) + issues_created = seeder.seed(backfill_weeks: args.backfill_weeks.to_i, + average_issues_per_week: args.average_issues_per_week.to_i) puts "\n#{issues_created} issues created!" end end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index e876b23d43f..5d86d6e466c 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -31,7 +31,6 @@ namespace :gitlab do terminate_all_connections unless Rails.env.production? Rake::Task["db:reset"].invoke - Rake::Task["add_limits_mysql"].invoke Rake::Task["setup_postgresql"].invoke Rake::Task["db:seed_fu"].invoke rescue Gitlab::TaskAbortedByUserError @@ -46,8 +45,6 @@ namespace :gitlab do # method terminates all the connections so that a subsequent DROP # will work. def self.terminate_all_connections - return false unless Gitlab::Database.postgresql? - cmd = <<~SQL SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index 954f827f716..ccc96b7edfb 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -3,50 +3,44 @@ namespace :gitlab do desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' task migrate_to_hashed: :environment do if Gitlab::Database.read_only? - warn 'This task requires database write access. Exiting.' - - next + abort 'This task requires database write access. Exiting.' end storage_migrator = Gitlab::HashedStorage::Migrator.new helper = Gitlab::HashedStorage::RakeHelper if storage_migrator.rollback_pending? - warn "There is already a rollback operation in progress, " \ + abort "There is already a rollback operation in progress, " \ "running a migration at the same time may have unexpected consequences." - - next end if helper.range_single_item? project = Project.with_unmigrated_storage.find_by(id: helper.range_from) unless project - warn "There are no projects requiring storage migration with ID=#{helper.range_from}" - - next + abort "There are no projects requiring storage migration with ID=#{helper.range_from}" end puts "Enqueueing storage migration of #{project.full_path} (ID=#{project.id})..." storage_migrator.migrate(project) + else + legacy_projects_count = if helper.using_ranges? + Project.with_unmigrated_storage.id_in(helper.range_from..helper.range_to).count + else + Project.with_unmigrated_storage.count + end + + if legacy_projects_count == 0 + abort 'There are no projects requiring storage migration. Nothing to do!' + end - next - end - - legacy_projects_count = Project.with_unmigrated_storage.count - - if legacy_projects_count == 0 - warn 'There are no projects requiring storage migration. Nothing to do!' - - next - end - - print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}" + print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}" - helper.project_id_batches_migration do |start, finish| - storage_migrator.bulk_schedule_migration(start: start, finish: finish) + helper.project_id_batches_migration do |start, finish| + storage_migrator.bulk_schedule_migration(start: start, finish: finish) - print '.' + print '.' + end end puts ' Done!' @@ -55,50 +49,44 @@ namespace :gitlab do desc 'GitLab | Storage | Rollback existing projects to Legacy Storage' task rollback_to_legacy: :environment do if Gitlab::Database.read_only? - warn 'This task requires database write access. Exiting.' - - next + abort 'This task requires database write access. Exiting.' end storage_migrator = Gitlab::HashedStorage::Migrator.new helper = Gitlab::HashedStorage::RakeHelper if storage_migrator.migration_pending? - warn "There is already a migration operation in progress, " \ + abort "There is already a migration operation in progress, " \ "running a rollback at the same time may have unexpected consequences." - - next end if helper.range_single_item? project = Project.with_storage_feature(:repository).find_by(id: helper.range_from) unless project - warn "There are no projects that can be rolledback with ID=#{helper.range_from}" - - next + abort "There are no projects that can be rolledback with ID=#{helper.range_from}" end puts "Enqueueing storage rollback of #{project.full_path} (ID=#{project.id})..." storage_migrator.rollback(project) + else + hashed_projects_count = if helper.using_ranges? + Project.with_storage_feature(:repository).id_in(helper.range_from..helper.range_to).count + else + Project.with_storage_feature(:repository).count + end + + if hashed_projects_count == 0 + abort 'There are no projects that can have storage rolledback. Nothing to do!' + end - next - end - - hashed_projects_count = Project.with_storage_feature(:repository).count - - if hashed_projects_count == 0 - warn 'There are no projects that can have storage rolledback. Nothing to do!' - - next - end - - print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}" + print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}" - helper.project_id_batches_rollback do |start, finish| - storage_migrator.bulk_schedule_rollback(start: start, finish: finish) + helper.project_id_batches_rollback do |start, finish| + storage_migrator.bulk_schedule_rollback(start: start, finish: finish) - print '.' + print '.' + end end puts ' Done!' diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index e058e9fe069..fdcd34320b1 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -5,25 +5,42 @@ namespace :gitlab do end desc "GitLab | Update project templates" - task :update_project_templates do - include Gitlab::ImportExport::CommandLineUtil + task :update_project_templates, [] => :environment do |_task, args| + # we need an instance method from Gitlab::ImportExport::CommandLineUtil and don't + # want to include it in the task, as this would affect subsequent tasks as well + downloader = Class.new do + extend Gitlab::ImportExport::CommandLineUtil + + def self.call(uploader, upload_path) + download_or_copy_upload(uploader, upload_path) + end + end + + template_names = args.extras.to_set if Rails.env.production? - puts "This rake task is not meant fo production instances".red - exit(1) + raise "This rake task is not meant for production instances" end admin = User.find_by(admin: true) unless admin - puts "No admin user could be found".red - exit(1) + raise "No admin user could be found" end - Gitlab::ProjectTemplate.all.each do |template| + tmp_namespace_path = "tmp-project-import-#{Time.now.to_i}" + puts "Creating temporary namespace #{tmp_namespace_path}" + tmp_namespace = Namespace.create!(owner: admin, name: tmp_namespace_path, path: tmp_namespace_path) + + templates = if template_names.empty? + Gitlab::ProjectTemplate.all + else + Gitlab::ProjectTemplate.all.select { |template| template_names.include?(template.name) } + end + + templates.each do |template| params = { - import_url: template.clone_url, - namespace_id: admin.namespace.id, + namespace_id: tmp_namespace.id, path: template.name, skip_wiki: true } @@ -32,33 +49,67 @@ namespace :gitlab do project = Projects::CreateService.new(admin, params).execute unless project.persisted? - puts project.errors.messages - exit(1) + raise "Failed to create project: #{project.errors.messages}" end - loop do - if project.finished? - puts "Import finished for #{template.name}" - break + uri_encoded_project_path = template.uri_encoded_project_path + + # extract a concrete commit for signing off what we actually downloaded + # this way we do the right thing even if the repository gets updated in the meantime + get_commits_response = Gitlab::HTTP.get("https://gitlab.com/api/v4/projects/#{uri_encoded_project_path}/repository/commits", + query: { page: 1, per_page: 1 } + ) + raise "Failed to retrieve latest commit for template '#{template.name}'" unless get_commits_response.success? + + commit_sha = get_commits_response.parsed_response.dig(0, 'id') + + project_archive_uri = "https://gitlab.com/api/v4/projects/#{uri_encoded_project_path}/repository/archive.tar.gz?sha=#{commit_sha}" + commit_message = <<~MSG + Initialized from '#{template.title}' project template + + Template repository: #{template.preview} + Commit SHA: #{commit_sha} + MSG + + Dir.mktmpdir do |tmpdir| + Dir.chdir(tmpdir) do + Gitlab::TaskHelpers.run_command!(['wget', project_archive_uri, '-O', 'archive.tar.gz']) + Gitlab::TaskHelpers.run_command!(['tar', 'xf', 'archive.tar.gz']) + extracted_project_basename = Dir['*/'].first + Dir.chdir(extracted_project_basename) do + Gitlab::TaskHelpers.run_command!(%w(git init)) + Gitlab::TaskHelpers.run_command!(%w(git add .)) + Gitlab::TaskHelpers.run_command!(['git', 'commit', '--author', 'GitLab <root@localhost>', '--message', commit_message]) + + # Hacky workaround to push to the project in a way that works with both GDK and the test environment + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab::TaskHelpers.run_command!(['git', 'remote', 'add', 'origin', "file://#{project.repository.raw.path}"]) + end + Gitlab::TaskHelpers.run_command!(['git', 'push', '-u', 'origin', 'master']) + end end + end - if project.failed? - puts "Failed to import from #{project_params[:import_url]}".red - exit(1) - end + project.reset - puts "Waiting for the import to finish" + Projects::ImportExport::ExportService.new(project, admin).execute + downloader.call(project.export_file, template.archive_path) - sleep(5) - project.reset + unless Projects::DestroyService.new(project, admin).execute + puts "Failed to destroy project #{template.name} (but namespace will be cleaned up later)" end - Projects::ImportExport::ExportService.new(project, admin).execute - download_or_copy_upload(project.export_file, template.archive_path) - Projects::DestroyService.new(admin, project).execute puts "Exported #{template.name}".green end - puts "Done".green + + success = true + ensure + if tmp_namespace + puts "Destroying temporary namespace #{tmp_namespace_path}" + tmp_namespace.destroy + end + + puts "Done".green if success end def update(template) diff --git a/lib/tasks/gitlab/uploads/legacy.rake b/lib/tasks/gitlab/uploads/legacy.rake new file mode 100644 index 00000000000..18fb8afe455 --- /dev/null +++ b/lib/tasks/gitlab/uploads/legacy.rake @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :uploads do + namespace :legacy do + desc "GitLab | Uploads | Migrate all legacy attachments" + task migrate: :environment do + class Upload < ApplicationRecord + self.table_name = 'uploads' + + include ::EachBatch + end + + migration = 'LegacyUploadsMigrator'.freeze + batch_size = 5000 + delay_interval = 5.minutes.to_i + + Upload.where(uploader: 'AttachmentUploader').each_batch(of: batch_size) do |relation, index| + start_id, end_id = relation.pluck('MIN(id), MAX(id)').first + delay = index * delay_interval + + BackgroundMigrationWorker.perform_in(delay, migration, [start_id, end_id]) + end + end + end + end +end diff --git a/lib/tasks/karma.rake b/lib/tasks/karma.rake index 2dc14183fa3..36590010406 100644 --- a/lib/tasks/karma.rake +++ b/lib/tasks/karma.rake @@ -1,15 +1,8 @@ unless Rails.env.production? namespace :karma do + # alias exists for legacy reasons desc 'GitLab | Karma | Generate fixtures for JavaScript tests' - task fixtures: ['karma:rspec_fixtures'] - - desc 'GitLab | Karma | Generate fixtures using RSpec' - RSpec::Core::RakeTask.new(:rspec_fixtures, [:pattern]) do |t, args| - args.with_defaults(pattern: '{spec,ee/spec}/javascripts/fixtures/*.rb') - ENV['NO_KNAPSACK'] = 'true' - t.pattern = args[:pattern] - t.rspec_opts = '--format documentation' - end + task fixtures: ['frontend:fixtures'] desc 'GitLab | Karma | Run JavaScript tests' task tests: ['yarn:check'] do diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake deleted file mode 100644 index c77fa49d586..00000000000 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ /dev/null @@ -1,17 +0,0 @@ -require Rails.root.join('db/migrate/limits_to_mysql') -require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') -require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql') -require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql') -require Rails.root.join('db/migrate/gpg_keys_limits_to_mysql') -require Rails.root.join('db/migrate/prometheus_metrics_limits_to_mysql') - -desc "GitLab | Add limits to strings in mysql database" -task add_limits_mysql: :environment do - puts "Adding limits to schema.rb for mysql" - LimitsToMysql.new.up - MarkdownCacheLimitsToMysql.new.up - MergeRequestDiffFileLimitsToMysql.new.up - LimitsCiBuildTraceChunksRawDataForMysql.new.up - IncreaseMysqlTextLimitForGpgKeys.new.up - PrometheusMetricsLimitsToMysql.new.up -end diff --git a/lib/tasks/services.rake b/lib/tasks/services.rake index 56b81106c5f..4ec4fdd281f 100644 --- a/lib/tasks/services.rake +++ b/lib/tasks/services.rake @@ -86,7 +86,7 @@ namespace :services do doc_start = Time.now doc_path = File.join(Rails.root, 'doc', 'api', 'services.md') - result = ERB.new(services_template, 0, '>') + result = ERB.new(services_template, trim_mode: '>') .result(OpenStruct.new(services: services).instance_eval { binding }) File.open(doc_path, 'w') do |f| diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index c881ad4cf12..bf18332a8eb 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -2,8 +2,6 @@ return if Rails.env.production? -Rake::Task["spec"].clear if Rake::Task.task_defined?('spec') - namespace :spec do desc 'GitLab | RSpec | Run unit tests' RSpec::Core::RakeTask.new(:unit, :rspec_opts) do |t, args| @@ -26,63 +24,8 @@ namespace :spec do t.rspec_opts = args[:rspec_opts] end - desc '[Deprecated] Use the "bin/rspec --tag api" instead' - task :api do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @api) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use the "spec:system" task instead' - task :feature do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @feature) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use "bin/rspec spec/models" instead' - task :models do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @models) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use "bin/rspec spec/services" instead' - task :services do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @services) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use "bin/rspec spec/lib" instead' - task :lib do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @lib) - ] - run_commands(cmds) - end -end - -desc "GitLab | Run specs" -task :spec do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec) - ] - run_commands(cmds) -end - -def run_commands(cmds) - cmds.each do |cmd| - system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) || raise("#{cmd} failed!") + desc 'Run the code examples in spec/requests/api' + RSpec::Core::RakeTask.new(:api) do |t| + t.pattern = 'spec/requests/api/**/*_spec.rb' end end |