diff options
Diffstat (limited to 'lib')
61 files changed, 685 insertions, 304 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index a5c9f0b509c..c9b5f58c557 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -68,8 +68,8 @@ module API delete ":id/access_requests/:user_id" do source = find_source(source_type, params[:id]) - ::Members::DestroyService.new(source, current_user, params). - execute(:requesters) + ::Members::DestroyService.new(source, current_user, params) + .execute(:requesters) end end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index f35084a582a..3d816f8771d 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -102,8 +102,8 @@ module API post ":id/repository/branches" do authorize_push_project - result = CreateBranchService.new(user_project, current_user). - execute(params[:branch], params[:ref]) + result = CreateBranchService.new(user_project, current_user) + .execute(params[:branch], params[:ref]) if result[:status] == :success present result[:branch], @@ -121,8 +121,8 @@ module API delete ":id/repository/branches/:branch", requirements: { branch: /.+/ } do authorize_push_project - result = DeleteBranchService.new(user_project, current_user). - execute(params[:branch]) + result = DeleteBranchService.new(user_project, current_user) + .execute(params[:branch]) if result[:status] != :success render_api_error!(result[:message], result[:return_code]) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 675bc52a983..aa91451c9f4 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -484,9 +484,9 @@ module API expose :job_events # Expose serialized properties expose :properties do |service, options| - field_names = service.fields. - select { |field| options[:include_passwords] || field[:type] != 'password' }. - map { |field| field[:name] } + field_names = service.fields + .select { |field| options[:include_passwords] || field[:type] != 'password' } + .map { |field| field[:name] } service.properties.slice(*field_names) end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index e281e3230fd..01ca62b593f 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -33,8 +33,8 @@ module API # paginate() only works with a relation. This could lead to a # mismatch between the pagination headers info and the actual notes # array returned, but this is really a edge-case. - paginate(noteable.notes). - reject { |n| n.cross_reference_not_visible_for?(current_user) } + paginate(noteable.notes) + .reject { |n| n.cross_reference_not_visible_for?(current_user) } present notes, with: Entities::Note else not_found!("Notes") diff --git a/lib/api/tags.rb b/lib/api/tags.rb index c7b1efe0bfa..633a858f8c7 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -44,8 +44,8 @@ module API post ':id/repository/tags' do authorize_push_project - result = ::Tags::CreateService.new(user_project, current_user). - execute(params[:tag_name], params[:ref], params[:message], params[:release_description]) + result = ::Tags::CreateService.new(user_project, current_user) + .execute(params[:tag_name], params[:ref], params[:message], params[:release_description]) if result[:status] == :success present result[:tag], @@ -63,8 +63,8 @@ module API delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do authorize_push_project - result = ::Tags::DestroyService.new(user_project, current_user). - execute(params[:tag_name]) + result = ::Tags::DestroyService.new(user_project, current_user) + .execute(params[:tag_name]) if result[:status] != :success render_api_error!(result[:message], result[:return_code]) @@ -81,8 +81,8 @@ module API post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project - result = CreateReleaseService.new(user_project, current_user). - execute(params[:tag_name], params[:description]) + result = CreateReleaseService.new(user_project, current_user) + .execute(params[:tag_name], params[:description]) if result[:status] == :success present result[:release], with: Entities::Release @@ -101,8 +101,8 @@ module API put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project - result = UpdateReleaseService.new(user_project, current_user). - execute(params[:tag_name], params[:description]) + result = UpdateReleaseService.new(user_project, current_user) + .execute(params[:tag_name], params[:description]) if result[:status] == :success present result[:release], with: Entities::Release diff --git a/lib/api/users.rb b/lib/api/users.rb index bfb69d6dc18..c10e3364382 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -103,13 +103,13 @@ module API if user.persisted? present user, with: Entities::UserPublic else - conflict!('Email has already been taken') if User. - where(email: user.email). - count > 0 + conflict!('Email has already been taken') if User + .where(email: user.email) + .count > 0 - conflict!('Username has already been taken') if User. - where(username: user.username). - count > 0 + conflict!('Username has already been taken') if User + .where(username: user.username) + .count > 0 render_validation_error!(user) end @@ -133,12 +133,12 @@ module API not_found!('User') unless user conflict!('Email has already been taken') if params[:email] && - User.where(email: params[:email]). - where.not(id: user.id).count > 0 + User.where(email: params[:email]) + .where.not(id: user.id).count > 0 conflict!('Username has already been taken') if params[:username] && - User.where(username: params[:username]). - where.not(id: user.id).count > 0 + User.where(username: params[:username]) + .where.not(id: user.id).count > 0 user_params = declared_params(include_missing: false) identity_attrs = user_params.slice(:provider, :extern_uid) @@ -517,9 +517,9 @@ module API get "activities" do authenticated_as_admin! - activities = User. - where(User.arel_table[:last_activity_on].gteq(params[:from])). - reorder(last_activity_on: :asc) + activities = User + .where(User.arel_table[:last_activity_on].gteq(params[:from])) + .reorder(last_activity_on: :asc) present paginate(activities), with: Entities::UserActivity end diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb index 0a877b960f6..81b13249892 100644 --- a/lib/api/v3/branches.rb +++ b/lib/api/v3/branches.rb @@ -26,8 +26,8 @@ module API delete ":id/repository/branches/:branch", requirements: { branch: /.+/ } do authorize_push_project - result = DeleteBranchService.new(user_project, current_user). - execute(params[:branch]) + result = DeleteBranchService.new(user_project, current_user) + .execute(params[:branch]) if result[:status] == :success status(200) @@ -55,8 +55,8 @@ module API end post ":id/repository/branches" do authorize_push_project - result = CreateBranchService.new(user_project, current_user). - execute(params[:branch_name], params[:ref]) + result = CreateBranchService.new(user_project, current_user) + .execute(params[:branch_name], params[:ref]) if result[:status] == :success present result[:branch], diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 7c5065dee90..c848f52723b 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -245,9 +245,9 @@ module API expose :job_events, as: :build_events # Expose serialized properties expose :properties do |service, options| - field_names = service.fields. - select { |field| options[:include_passwords] || field[:type] != 'password' }. - map { |field| field[:name] } + field_names = service.fields + .select { |field| options[:include_passwords] || field[:type] != 'password' } + .map { |field| field[:name] } service.properties.slice(*field_names) end end diff --git a/lib/api/v3/helpers.rb b/lib/api/v3/helpers.rb index d9e76560d03..4e63aa01c1a 100644 --- a/lib/api/v3/helpers.rb +++ b/lib/api/v3/helpers.rb @@ -38,7 +38,10 @@ module API projects = projects.where(visibility_level: Gitlab::VisibilityLevel.level_value(params[:visibility])) end - projects = projects.where(archived: params[:archived]) + unless params[:archived].nil? + projects = projects.where(archived: to_boolean(params[:archived])) + end + projects.reorder(params[:order_by] => params[:sort]) end end diff --git a/lib/api/v3/notes.rb b/lib/api/v3/notes.rb index 009ec5c6bbd..23fe95e42e4 100644 --- a/lib/api/v3/notes.rb +++ b/lib/api/v3/notes.rb @@ -34,8 +34,8 @@ module API # paginate() only works with a relation. This could lead to a # mismatch between the pagination headers info and the actual notes # array returned, but this is really a edge-case. - paginate(noteable.notes). - reject { |n| n.cross_reference_not_visible_for?(current_user) } + paginate(noteable.notes) + .reject { |n| n.cross_reference_not_visible_for?(current_user) } present notes, with: ::API::V3::Entities::Note else not_found!("Notes") diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 20976b9dd08..eb090453b48 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -69,7 +69,7 @@ module API end params :filter_params do - optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :archived, type: Boolean, default: nil, desc: 'Limit by archived status' optional :visibility, type: String, values: %w[public internal private], desc: 'Limit by visibility' optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' diff --git a/lib/api/v3/tags.rb b/lib/api/v3/tags.rb index c2541de2f50..7e5875cd030 100644 --- a/lib/api/v3/tags.rb +++ b/lib/api/v3/tags.rb @@ -22,8 +22,8 @@ module API delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do authorize_push_project - result = ::Tags::DestroyService.new(user_project, current_user). - execute(params[:tag_name]) + result = ::Tags::DestroyService.new(user_project, current_user) + .execute(params[:tag_name]) if result[:status] == :success status(200) diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb index f4cda3b2eba..37020019e07 100644 --- a/lib/api/v3/users.rb +++ b/lib/api/v3/users.rb @@ -50,13 +50,13 @@ module API if user.persisted? present user, with: ::API::Entities::UserPublic else - conflict!('Email has already been taken') if User. - where(email: user.email). - count > 0 + conflict!('Email has already been taken') if User + .where(email: user.email) + .count > 0 - conflict!('Username has already been taken') if User. - where(username: user.username). - count > 0 + conflict!('Username has already been taken') if User + .where(username: user.username) + .count > 0 render_validation_error!(user) end @@ -137,11 +137,11 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user - events = user.events. - merge(ProjectsFinder.new(current_user: current_user).execute). - references(:project). - with_associations. - recent + events = user.events + .merge(ProjectsFinder.new(current_user: current_user).execute) + .references(:project) + .with_associations + .recent present paginate(events), with: ::API::V3::Entities::Event end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index 8e3b0c4db79..7e6357f8a00 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -10,8 +10,8 @@ module Banzai end def references(type, project, current_user = nil) - processor = Banzai::ReferenceParser[type]. - new(project, current_user) + processor = Banzai::ReferenceParser[type] + .new(project, current_user) processor.process(html_documents) end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 89ec715ddf6..9fd4bd68d43 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -9,8 +9,8 @@ module Banzai issues = issues_for_nodes(nodes) - readable_issues = Ability. - issues_readable_by_user(issues.values, user).to_set + readable_issues = Ability + .issues_readable_by_user(issues.values, user).to_set nodes.select do |node| readable_issues.include?(issues[node]) diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index 3efbd2fd631..4d336068861 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -99,8 +99,8 @@ module Banzai def find_users_for_projects(ids) return [] if ids.empty? - collection_objects_for_ids(Project, ids). - flat_map { |p| p.team.members.to_a } + collection_objects_for_ids(Project, ids) + .flat_map { |p| p.team.members.to_a } end def can_read_reference?(user, ref_project, node) diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index 3decc3b1a26..6063d6f45e8 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -2,10 +2,10 @@ module Ci module Charts module DailyInterval def grouped_count(query) - query. - group("DATE(#{Ci::Build.table_name}.created_at)"). - count(:created_at). - transform_keys { |date| date.strftime(@format) } + query + .group("DATE(#{Ci::Build.table_name}.created_at)") + .count(:created_at) + .transform_keys { |date| date.strftime(@format) } end def interval_step @@ -16,14 +16,14 @@ module Ci module MonthlyInterval def grouped_count(query) if Gitlab::Database.postgresql? - query. - group("to_char(#{Ci::Build.table_name}.created_at, '01 Month YYYY')"). - count(:created_at). - transform_keys(&:squish) + query + .group("to_char(#{Ci::Build.table_name}.created_at, '01 Month YYYY')") + .count(:created_at) + .transform_keys(&:squish) else - query. - group("DATE_FORMAT(#{Ci::Build.table_name}.created_at, '01 %M %Y')"). - count(:created_at) + query + .group("DATE_FORMAT(#{Ci::Build.table_name}.created_at, '01 %M %Y')") + .count(:created_at) end end @@ -46,8 +46,8 @@ module Ci end def collect - query = project.builds. - where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", @to, @from) + query = project.builds + .where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", @to, @from) totals_count = grouped_count(query) success_count = grouped_count(query.success) diff --git a/lib/feature.rb b/lib/feature.rb index 5650a1c1334..d3d972564af 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -39,8 +39,6 @@ class Feature get(key).disable end - private - def flipper @flipper ||= begin adapter = Flipper::Adapters::ActiveRecord.new( diff --git a/lib/github/import.rb b/lib/github/import.rb index b20614b3060..ff5d7db2705 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -172,7 +172,7 @@ module Github next unless merge_request.new_record? && pull_request.valid? begin - restore_branches(pull_request) + pull_request.restore_branches! author_id = user_id(pull_request.author, project.creator_id) description = format_description(pull_request.description, pull_request.author) @@ -208,7 +208,7 @@ module Github rescue => e error(:pull_request, pull_request.url, e.message) ensure - clean_up_restored_branches(pull_request) + pull_request.remove_restored_branches! end end @@ -325,32 +325,6 @@ module Github end end - def restore_branches(pull_request) - restore_source_branch(pull_request) unless pull_request.source_branch_exists? - restore_target_branch(pull_request) unless pull_request.target_branch_exists? - end - - def restore_source_branch(pull_request) - repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) - end - - def restore_target_branch(pull_request) - repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha) - end - - def remove_branch(name) - repository.delete_branch(name) - rescue Rugged::ReferenceError - errors << { type: :branch, url: nil, error: "Could not clean up restored branch: #{name}" } - end - - def clean_up_restored_branches(pull_request) - return if pull_request.opened? - - remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? - remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? - end - def label_ids(labels) labels.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb index d1dac6944f0..c6fa928d565 100644 --- a/lib/github/representation/branch.rb +++ b/lib/github/representation/branch.rb @@ -26,13 +26,25 @@ module Github end def exists? - branch_exists? && commit_exists? + @exists ||= branch_exists? && commit_exists? end def valid? sha.present? && ref.present? end + def restore!(name) + repository.create_branch(name, sha) + rescue Gitlab::Git::Repository::InvalidRef => e + Rails.logger.error("#{self.class.name}: Could not restore branch #{name}: #{e}") + end + + def remove!(name) + repository.delete_branch(name) + rescue Rugged::ReferenceError => e + Rails.logger.error("#{self.class.name}: Could not remove branch #{name}: #{e}") + end + private def branch_exists? diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb index ac9c8283b4b..55461097e8a 100644 --- a/lib/github/representation/pull_request.rb +++ b/lib/github/representation/pull_request.rb @@ -1,8 +1,6 @@ module Github module Representation class PullRequest < Representation::Issuable - attr_reader :project - delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true @@ -10,10 +8,6 @@ module Github project end - def source_branch_exists? - !cross_project? && source_branch.exists? - end - def source_branch_name @source_branch_name ||= if cross_project? || !source_branch_exists? @@ -23,6 +17,12 @@ module Github end end + def source_branch_exists? + return @source_branch_exists if defined?(@source_branch_exists) + + @source_branch_exists = !cross_project? && source_branch.exists? + end + def target_project project end @@ -31,6 +31,10 @@ module Github @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed end + def target_branch_exists? + @target_branch_exists ||= target_branch.exists? + end + def state return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present? return 'closed' if raw['state'] == 'closed' @@ -46,6 +50,18 @@ module Github source_branch.valid? && target_branch.valid? end + def restore_branches! + restore_source_branch! + restore_target_branch! + end + + def remove_restored_branches! + return if opened? + + remove_source_branch! + remove_target_branch! + end + private def project @@ -73,6 +89,32 @@ module Github source_branch_repo.id != target_branch_repo.id end + + def restore_source_branch! + return if source_branch_exists? + + source_branch.restore!(source_branch_name) + end + + def restore_target_branch! + return if target_branch_exists? + + target_branch.restore!(target_branch_name) + end + + def remove_source_branch! + # We should remove the source/target branches only if they were + # restored. Otherwise, we'll remove branches like 'master' that + # target_branch_exists? returns true. In other words, we need + # to clean up only the restored branches that (source|target)_branch_exists? + # returns false for the first time it has been called, because of + # this that is important to memoize these values. + source_branch.remove!(source_branch_name) unless source_branch_exists? + end + + def remove_target_branch! + target_branch.remove!(target_branch_name) unless target_branch_exists? + end end end end diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index 914a3b72abd..d95ecd7b291 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -5,8 +5,8 @@ module Gitlab # # steal_class - The name of the class for which to steal jobs. def self.steal(steal_class) - queue = Sidekiq::Queue. - new(BackgroundMigrationWorker.sidekiq_options['queue']) + queue = Sidekiq::Queue + .new(BackgroundMigrationWorker.sidekiq_options['queue']) queue.each do |job| migration_class, migration_args = job.args diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 4fc9a075edc..9c2e09943b0 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -50,8 +50,8 @@ module Gitlab ref: pipeline.ref } - new(pipeline.project, pipeline_info: pipeline_info). - store_in_cache_if_needed + new(pipeline.project, pipeline_info: pipeline_info) + .store_in_cache_if_needed end def initialize(project, pipeline_info: {}, loaded_from_cache: nil) diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb index a210e76acaa..3208cc2bef6 100644 --- a/lib/gitlab/ci/pipeline_duration.rb +++ b/lib/gitlab/ci/pipeline_duration.rb @@ -87,8 +87,8 @@ module Gitlab def from_pipeline(pipeline) status = %w[success failed running canceled] - builds = pipeline.builds.latest. - where(status: status).where.not(started_at: nil).order(:started_at) + builds = pipeline.builds.latest + .where(status: status).where.not(started_at: nil).order(:started_at) from_builds(builds) end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 6e73361cad1..1611eba31da 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -16,9 +16,9 @@ module Gitlab project = merge_request.source_project new(merge_request, project).tap do |file_collection| - project. - repository. - with_repo_branch_commit(merge_request.target_project.repository, merge_request.target_branch) do + project + .repository + .with_repo_branch_commit(merge_request.target_project.repository, merge_request.target_branch) do yield file_collection end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 060e013183f..bf557103cfd 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -16,14 +16,14 @@ module Gitlab # Can't use Event.contributions here because we need to check 3 different # project_features for the (currently) 3 different contribution types date_from = 1.year.ago - repo_events = event_counts(date_from, :repository). - having(action: Event::PUSHED) - issue_events = event_counts(date_from, :issues). - having(action: [Event::CREATED, Event::CLOSED], target_type: "Issue") - mr_events = event_counts(date_from, :merge_requests). - having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest") - note_events = event_counts(date_from, :merge_requests). - having(action: [Event::COMMENTED], target_type: "Note") + repo_events = event_counts(date_from, :repository) + .having(action: Event::PUSHED) + issue_events = event_counts(date_from, :issues) + .having(action: [Event::CREATED, Event::CLOSED], target_type: "Issue") + mr_events = event_counts(date_from, :merge_requests) + .having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest") + note_events = event_counts(date_from, :merge_requests) + .having(action: [Event::COMMENTED], target_type: "Note") union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) events = Event.find_by_sql(union.to_sql).map(&:attributes) @@ -34,9 +34,9 @@ module Gitlab end def events_by_date(date) - events = Event.contributions.where(author_id: contributor.id). - where(created_at: date.beginning_of_day..date.end_of_day). - where(project_id: projects) + events = Event.contributions.where(author_id: contributor.id) + .where(created_at: date.beginning_of_day..date.end_of_day) + .where(project_id: projects) # Use visible_to_user? instead of the complicated logic in activity_dates # because we're only viewing the events for a single day. @@ -60,20 +60,20 @@ module Gitlab # use IN(project_ids...) instead. It's the intersection of two users so # the list will be (relatively) short @contributed_project_ids ||= projects.uniq.pluck(:id) - authed_projects = Project.where(id: @contributed_project_ids). - with_feature_available_for_user(feature, current_user). - reorder(nil). - select(:id) + authed_projects = Project.where(id: @contributed_project_ids) + .with_feature_available_for_user(feature, current_user) + .reorder(nil) + .select(:id) - conditions = t[:created_at].gteq(date_from.beginning_of_day). - and(t[:created_at].lteq(Date.today.end_of_day)). - and(t[:author_id].eq(contributor.id)) + conditions = t[:created_at].gteq(date_from.beginning_of_day) + .and(t[:created_at].lteq(Date.today.end_of_day)) + .and(t[:author_id].eq(contributor.id)) - Event.reorder(nil). - select(t[:project_id], t[:target_type], t[:action], 'date(created_at) AS date', 'count(id) as total_amount'). - group(t[:project_id], t[:target_type], t[:action], 'date(created_at)'). - where(conditions). - having(t[:project_id].in(Arel::Nodes::SqlLiteral.new(authed_projects.to_sql))) + Event.reorder(nil) + .select(t[:project_id], t[:target_type], t[:action], 'date(created_at) AS date', 'count(id) as total_amount') + .group(t[:project_id], t[:target_type], t[:action], 'date(created_at)') + .where(conditions) + .having(t[:project_id].in(Arel::Nodes::SqlLiteral.new(authed_projects.to_sql))) end end end diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index d560dca45c8..58729d3ced8 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -12,17 +12,17 @@ module Gitlab end def stage_query - query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])). - join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])). - where(issue_table[:project_id].eq(@project.id)). - where(issue_table[:deleted_at].eq(nil)). - where(issue_table[:created_at].gteq(@options[:from])) + query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])) + .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) + .where(issue_table[:project_id].eq(@project.id)) + .where(issue_table[:deleted_at].eq(nil)) + .where(issue_table[:created_at].gteq(@options[:from])) # Load merge_requests - query = query.join(mr_table, Arel::Nodes::OuterJoin). - on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])). - join(mr_metrics_table). - on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) + query = query.join(mr_table, Arel::Nodes::OuterJoin) + .on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])) + .join(mr_metrics_table) + .on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) query end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 23890e5f493..059054ac9ff 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -29,10 +29,10 @@ module Gitlab end def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - query = arel_table. - from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)). - project(average([arel_table[column_sym]], 'median')). - where( + query = arel_table + .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) + .project(average([arel_table[column_sym]], 'median')) + .where( Arel::Nodes::Between.new( Arel.sql("(select @row_id := @row_id + 1)"), Arel::Nodes::And.new( @@ -67,8 +67,8 @@ module Gitlab cte_table = Arel::Table.new("ordered_records") cte = Arel::Nodes::As.new( cte_table, - arel_table. - project( + arel_table + .project( arel_table[column_sym].as(column_sym.to_s), Arel::Nodes::Over.new(Arel::Nodes::NamedFunction.new("row_number", []), Arel::Nodes::Window.new.order(arel_table[column_sym])).as('row_id'), @@ -79,8 +79,8 @@ module Gitlab # From the CTE, select either the middle row or the middle two rows (this is accomplished # by 'where cte.row_id between cte.ct / 2.0 AND cte.ct / 2.0 + 1'). Find the average of the # selected rows, and this is the median value. - cte_table.project(average([extract_epoch(cte_table[column_sym])], "median")). - where( + cte_table.project(average([extract_epoch(cte_table[column_sym])], "median")) + .where( Arel::Nodes::Between.new( cte_table[:row_id], Arel::Nodes::And.new( @@ -88,9 +88,9 @@ module Gitlab (cte_table[:ct] / Arel.sql('2.0') + 1)] ) ) - ). - with(query_so_far, cte). - to_sql + ) + .with(query_so_far, cte) + .to_sql end private diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 9181202a091..60cce9c6d9e 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -245,19 +245,19 @@ module Gitlab start_id = exec_query(start_arel.to_sql).to_hash.first['id'].to_i loop do - stop_arel = table.project(table[:id]). - where(table[:id].gteq(start_id)). - order(table[:id].asc). - take(1). - skip(batch_size) + stop_arel = table.project(table[:id]) + .where(table[:id].gteq(start_id)) + .order(table[:id].asc) + .take(1) + .skip(batch_size) stop_arel = yield table, stop_arel if block_given? stop_row = exec_query(stop_arel.to_sql).to_hash.first - update_arel = Arel::UpdateManager.new(ActiveRecord::Base). - table(table). - set([[table[column], value]]). - where(table[:id].gteq(start_id)) + update_arel = Arel::UpdateManager.new(ActiveRecord::Base) + .table(table) + .set([[table[column], value]]) + .where(table[:id].gteq(start_id)) if stop_row stop_id = stop_row['id'].to_i @@ -586,15 +586,15 @@ module Gitlab quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) if Database.mysql? - locate = Arel::Nodes::NamedFunction. - new('locate', [quoted_pattern, column]) - insert_in_place = Arel::Nodes::NamedFunction. - new('insert', [column, locate, pattern.size, quoted_replacement]) + locate = Arel::Nodes::NamedFunction + .new('locate', [quoted_pattern, column]) + insert_in_place = Arel::Nodes::NamedFunction + .new('insert', [column, locate, pattern.size, quoted_replacement]) Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) else - replace = Arel::Nodes::NamedFunction. - new("regexp_replace", [column, quoted_pattern, quoted_replacement]) + replace = Arel::Nodes::NamedFunction + .new("regexp_replace", [column, quoted_pattern, quoted_replacement]) Arel::Nodes::SqlLiteral.new(replace.to_sql) end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index d60fd4bb551..d8163d7da11 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -27,8 +27,8 @@ module Gitlab new_full_path = join_routable_path(namespace_path, new_path) # skips callbacks & validations - routable.class.where(id: routable). - update_all(path: new_path) + routable.class.where(id: routable) + .update_all(path: new_path) rename_routes(old_full_path, new_full_path) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 2958ad4b8e5..da7e2cb2e85 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -18,8 +18,8 @@ module Gitlab when :top_level MigrationClasses::Namespace.where(parent_id: nil) end - with_paths = MigrationClasses::Route.arel_table[:path]. - matches_any(path_patterns) + with_paths = MigrationClasses::Route.arel_table[:path] + .matches_any(path_patterns) namespaces.joins(:route).where(with_paths) end @@ -52,15 +52,15 @@ module Gitlab end def repo_paths_for_namespace(namespace) - projects_for_namespace(namespace).distinct.select(:repository_storage). - map(&:repository_storage_path) + projects_for_namespace(namespace).distinct.select(:repository_storage) + .map(&:repository_storage_path) end def projects_for_namespace(namespace) namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id]) - namespace_or_children = MigrationClasses::Project. - arel_table[:namespace_id]. - in(namespace_ids) + namespace_or_children = MigrationClasses::Project + .arel_table[:namespace_id] + .in(namespace_ids) MigrationClasses::Project.where(namespace_or_children) end diff --git a/lib/gitlab/downtime_check.rb b/lib/gitlab/downtime_check.rb index ab9537ed7d7..941244694e2 100644 --- a/lib/gitlab/downtime_check.rb +++ b/lib/gitlab/downtime_check.rb @@ -50,8 +50,8 @@ module Gitlab # Returns the class for the given migration file path. def class_for_migration_file(path) - File.basename(path, File.extname(path)).split('_', 2).last.camelize. - constantize + File.basename(path, File.extname(path)).split('_', 2).last.camelize + .constantize end # Returns true if the given migration can be performed without downtime. diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 33a7624e303..a7aceab4c14 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -14,6 +14,51 @@ module Gitlab class << self def find(repository, sha, path) + Gitlab::GitalyClient.migrate(:project_raw_show) do |is_enabled| + if is_enabled + find_by_gitaly(repository, sha, path) + else + find_by_rugged(repository, sha, path) + end + end + end + + def find_by_gitaly(repository, sha, path) + path = path.sub(/\A\/*/, '') + path = '/' if path.empty? + name = File.basename(path) + entry = Gitlab::GitalyClient::Commit.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) + return unless entry + + case entry.type + when :COMMIT + new( + id: entry.oid, + name: name, + size: 0, + data: '', + path: path, + commit_id: sha + ) + when :BLOB + # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks + # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), + # which is what we use below to keep a consistent behavior. + detect = CharlockHolmes::EncodingDetector.new(8000).detect(entry.data) + new( + id: entry.oid, + name: name, + size: entry.size, + data: entry.data.dup, + mode: entry.mode.to_s(8), + path: path, + commit_id: sha, + binary: detect && detect[:type] == :binary + ) + end + end + + def find_by_rugged(repository, sha, path) commit = repository.lookup(sha) root_tree = commit.tree diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index f825568f194..cf7829a583b 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -318,7 +318,7 @@ module Gitlab end def init_from_gitaly(diff) - @diff = diff.patch if diff.respond_to?(:patch) + @diff = encode!(diff.patch) if diff.respond_to?(:patch) @new_path = encode!(diff.to_path.dup) @old_path = encode!(diff.from_path.dup) @a_mode = diff.old_mode.to_s(8) diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index 73c1848c95f..b8877619797 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -36,6 +36,26 @@ module Gitlab end end + def tree_entry(ref, path, limit = nil) + request = Gitaly::TreeEntryRequest.new( + repository: @gitaly_repo, + revision: ref, + path: path.dup.force_encoding(Encoding::ASCII_8BIT), + limit: limit.to_i + ) + + response = GitalyClient.call(@repository.storage, :commit, :tree_entry, request) + entry = response.first + return unless entry.oid.present? + + if entry.type == :BLOB + rest_of_data = response.reduce("") { |memo, msg| memo << msg.data } + entry.data += rest_of_data + end + + entry + end + private def commit_diff_request_params(commit, options = {}) diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb index d84e8d752dc..65d81dc5d46 100644 --- a/lib/gitlab/gitaly_client/diff_stitcher.rb +++ b/lib/gitlab/gitaly_client/diff_stitcher.rb @@ -13,7 +13,10 @@ module Gitlab @rpc_response.each do |diff_msg| if current_diff.nil? diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::FIELDS) - diff_params[:patch] = diff_msg.raw_patch_data + # gRPC uses frozen strings by default, and we need to have an unfrozen string as it + # gets processed further down the line. So we unfreeze the first chunk of the patch + # in case it's the only chunk we receive for this diff. + diff_params[:patch] = diff_msg.raw_patch_data.dup current_diff = GitalyClient::Diff.new(diff_params) else diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb index 357c076e874..5a31e56cb30 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/group_hierarchy.rb @@ -67,11 +67,11 @@ module Gitlab union = SQL::Union.new([model.unscoped.from(ancestors_table), model.unscoped.from(descendants_table)]) - model. - unscoped. - with. - recursive(ancestors.to_arel, descendants.to_arel). - from("(#{union.to_sql}) #{model.table_name}") + model + .unscoped + .with + .recursive(ancestors.to_arel, descendants.to_arel) + .from("(#{union.to_sql}) #{model.table_name}") end private @@ -82,10 +82,10 @@ module Gitlab cte << ancestors_base.except(:order) # Recursively get all the ancestors of the base set. - cte << model. - from([groups_table, cte.table]). - where(groups_table[:id].eq(cte.table[:parent_id])). - except(:order) + cte << model + .from([groups_table, cte.table]) + .where(groups_table[:id].eq(cte.table[:parent_id])) + .except(:order) cte end @@ -96,10 +96,10 @@ module Gitlab cte << descendants_base.except(:order) # Recursively get all the descendants of the base set. - cte << model. - from([groups_table, cte.table]). - where(groups_table[:parent_id].eq(cte.table[:id])). - except(:order) + cte << model + .from([groups_table, cte.table]) + .where(groups_table[:parent_id].eq(cte.table[:id])) + .except(:order) cte end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 6b24da030df..5408a1a6838 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,8 +1,8 @@ module Gitlab class Highlight def self.highlight(blob_name, blob_content, repository: nil, plain: false) - new(blob_name, blob_content, repository: repository). - highlight(blob_content, continue: false, plain: plain) + new(blob_name, blob_content, repository: repository) + .highlight(blob_content, continue: false, plain: plain) end attr_reader :blob_name diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 5e299e26c54..39180dc17d9 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -10,9 +10,9 @@ module Gitlab class << self def find_by_uid_and_provider(uid, provider) # LDAP distinguished name is case-insensitive - identity = ::Identity. - where(provider: provider). - iwhere(extern_uid: uid).last + identity = ::Identity + .where(provider: provider) + .iwhere(extern_uid: uid).last identity && identity.user end end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index 3a39791edbf..d7c56463aac 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -157,8 +157,8 @@ module Gitlab host = settings[:host] port = settings[:port] - InfluxDB::Client. - new(udp: { host: host, port: port }) + InfluxDB::Client + .new(udp: { host: host, port: port }) end end end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 3aaebb3e9c3..aba3e0df382 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -34,13 +34,13 @@ module Gitlab # THREAD_CPUTIME is not supported on OS X if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID) def self.cpu_time - Process. - clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) + Process + .clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) end else def self.cpu_time - Process. - clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) + Process + .clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) end end diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index 31a24460f0f..fc3f21233dd 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -6,8 +6,8 @@ module Gitlab # input - the source text in a markup format # def self.render(file_name, input, context) - html = GitHub::Markup.render(file_name, input). - force_encoding(input.encoding) + html = GitHub::Markup.render(file_name, input) + .force_encoding(input.encoding) context[:pipeline] = :markup html = Banzai.render(html, context) diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index 7ab80f5ee0f..574ae8731a5 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -3,8 +3,8 @@ module Gitlab module PerformanceBar module PeekQueryTracker def sorted_queries - PEEK_DB_CLIENT.query_details. - sort { |a, b| b[:duration] <=> a[:duration] } + PEEK_DB_CLIENT.query_details + .sort { |a, b| b[:duration] <=> a[:duration] } end def results diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb index bb0df1e3dad..15b8beacf60 100644 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ b/lib/gitlab/project_authorizations/with_nested_groups.rb @@ -28,34 +28,34 @@ module Gitlab # Projects that belong directly to any of the groups the user has # access to. - Namespace. - unscoped. - select([alias_as_column(projects[:id], 'project_id'), - cte_alias[:access_level]]). - from(cte_alias). - joins(:projects), + Namespace + .unscoped + .select([alias_as_column(projects[:id], 'project_id'), + cte_alias[:access_level]]) + .from(cte_alias) + .joins(:projects), # Projects shared with any of the namespaces the user has access to. - Namespace. - unscoped. - select([links[:project_id], - least(cte_alias[:access_level], - links[:group_access], - 'access_level')]). - from(cte_alias). - joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id'). - joins('INNER JOIN projects ON projects.id = project_group_links.project_id'). - joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id'). - where('p_ns.share_with_group_lock IS FALSE') + Namespace + .unscoped + .select([links[:project_id], + least(cte_alias[:access_level], + links[:group_access], + 'access_level')]) + .from(cte_alias) + .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') + .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') + .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') + .where('p_ns.share_with_group_lock IS FALSE') ] union = Gitlab::SQL::Union.new(relations) - ProjectAuthorization. - unscoped. - with. - recursive(cte.to_arel). - select_from_union(union) + ProjectAuthorization + .unscoped + .with + .recursive(cte.to_arel) + .select_from_union(union) end private @@ -68,17 +68,17 @@ module Gitlab namespaces = Namespace.arel_table # Namespaces the user is a member of. - cte << user.groups. - select([namespaces[:id], members[:access_level]]). - except(:order) + cte << user.groups + .select([namespaces[:id], members[:access_level]]) + .except(:order) # Sub groups of any groups the user is a member of. cte << Group.select([namespaces[:id], greatest(members[:access_level], - cte.table[:access_level], 'access_level')]). - joins(join_cte(cte)). - joins(join_members). - except(:order) + cte.table[:access_level], 'access_level')]) + .joins(join_cte(cte)) + .joins(join_members) + .except(:order) cte end @@ -88,11 +88,11 @@ module Gitlab members = Member.arel_table namespaces = Namespace.arel_table - cond = members[:source_id]. - eq(namespaces[:id]). - and(members[:source_type].eq('Namespace')). - and(members[:requested_at].eq(nil)). - and(members[:user_id].eq(user.id)) + cond = members[:source_id] + .eq(namespaces[:id]) + .and(members[:source_type].eq('Namespace')) + .and(members[:requested_at].eq(nil)) + .and(members[:user_id].eq(user.id)) Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) end diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb index 627e8c5fba2..ad87540e6c2 100644 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ b/lib/gitlab/project_authorizations/without_nested_groups.rb @@ -26,9 +26,9 @@ module Gitlab union = Gitlab::SQL::Union.new(relations) - ProjectAuthorization. - unscoped. - select_from_union(union) + ProjectAuthorization + .unscoped + .select_from_union(union) end end end diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb new file mode 100644 index 00000000000..cb95daf2260 --- /dev/null +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -0,0 +1,34 @@ +module Gitlab + module Prometheus + module AdditionalMetricsParser + extend self + + def load_groups_from_yaml + additional_metrics_raw.map(&method(:group_from_entry)) + end + + private + + def validate!(obj) + raise ParsingError.new(obj.errors.full_messages.join('\n')) unless obj.valid? + end + + def group_from_entry(entry) + entry[:name] = entry.delete(:group) + entry[:metrics]&.map! do |entry| + Metric.new(entry).tap(&method(:validate!)) + end + + MetricGroup.new(entry).tap(&method(:validate!)) + end + + def additional_metrics_raw + load_yaml_file&.map(&:deep_symbolize_keys).freeze + end + + def load_yaml_file + @loaded_yaml_file ||= YAML.load_file(Rails.root.join('config/prometheus/additional_metrics.yml')) + end + end + end +end diff --git a/lib/gitlab/prometheus/metric.rb b/lib/gitlab/prometheus/metric.rb new file mode 100644 index 00000000000..f54b2c6aaff --- /dev/null +++ b/lib/gitlab/prometheus/metric.rb @@ -0,0 +1,16 @@ +module Gitlab + module Prometheus + class Metric + include ActiveModel::Model + + attr_accessor :title, :required_metrics, :weight, :y_label, :queries + + validates :title, :required_metrics, :weight, :y_label, :queries, presence: true + + def initialize(params = {}) + super(params) + @y_label ||= 'Values' + end + end + end +end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb new file mode 100644 index 00000000000..729fef34b35 --- /dev/null +++ b/lib/gitlab/prometheus/metric_group.rb @@ -0,0 +1,14 @@ +module Gitlab + module Prometheus + class MetricGroup + include ActiveModel::Model + + attr_accessor :name, :priority, :metrics + validates :name, :priority, :metrics, presence: true + + def self.all + AdditionalMetricsParser.load_groups_from_yaml + end + end + end +end diff --git a/lib/gitlab/prometheus/parsing_error.rb b/lib/gitlab/prometheus/parsing_error.rb new file mode 100644 index 00000000000..49cc0e16080 --- /dev/null +++ b/lib/gitlab/prometheus/parsing_error.rb @@ -0,0 +1,5 @@ +module Gitlab + module Prometheus + ParsingError = Class.new(StandardError) + end +end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb new file mode 100644 index 00000000000..67c69d9ccf3 --- /dev/null +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -0,0 +1,22 @@ +module Gitlab + module Prometheus + module Queries + class AdditionalMetricsDeploymentQuery < BaseQuery + include QueryAdditionalMetrics + + def query(deployment_id) + Deployment.find_by(id: deployment_id).try do |deployment| + query_context = { + environment_slug: deployment.environment.slug, + environment_filter: %{container_name!="POD",environment="#{deployment.environment.slug}"}, + timeframe_start: (deployment.created_at - 30.minutes).to_f, + timeframe_end: (deployment.created_at + 30.minutes).to_f + } + + query_metrics(query_context) + end + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb new file mode 100644 index 00000000000..b5a679ddd79 --- /dev/null +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -0,0 +1,22 @@ +module Gitlab + module Prometheus + module Queries + class AdditionalMetricsEnvironmentQuery < BaseQuery + include QueryAdditionalMetrics + + def query(environment_id) + Environment.find_by(id: environment_id).try do |environment| + query_context = { + environment_slug: environment.slug, + environment_filter: %{container_name!="POD",environment="#{environment.slug}"}, + timeframe_start: 8.hours.ago.to_f, + timeframe_end: Time.now.to_f + } + + query_metrics(query_context) + end + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb index 2a2eb4ae57f..c60828165bd 100644 --- a/lib/gitlab/prometheus/queries/base_query.rb +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -3,7 +3,7 @@ module Gitlab module Queries class BaseQuery attr_accessor :client - delegate :query_range, :query, to: :client, prefix: true + delegate :query_range, :query, :label_values, :series, to: :client, prefix: true def raw_memory_usage_query(environment_slug) %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20} diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index 2cc08731f8d..170f483540e 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -1,26 +1,31 @@ -module Gitlab::Prometheus::Queries - class DeploymentQuery < BaseQuery - def query(deployment_id) - deployment = Deployment.find_by(id: deployment_id) - environment_slug = deployment.environment.slug +module Gitlab + module Prometheus + module Queries + class DeploymentQuery < BaseQuery + def query(deployment_id) + Deployment.find_by(id: deployment_id).try do |deployment| + environment_slug = deployment.environment.slug - memory_query = raw_memory_usage_query(environment_slug) - memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} - cpu_query = raw_cpu_usage_query(environment_slug) - cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} + memory_query = raw_memory_usage_query(environment_slug) + memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} + cpu_query = raw_cpu_usage_query(environment_slug) + cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} - timeframe_start = (deployment.created_at - 30.minutes).to_f - timeframe_end = (deployment.created_at + 30.minutes).to_f + timeframe_start = (deployment.created_at - 30.minutes).to_f + timeframe_end = (deployment.created_at + 30.minutes).to_f - { - memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), - memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), - memory_after: client_query(memory_avg_query, time: timeframe_end), + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), + memory_after: client_query(memory_avg_query, time: timeframe_end), - cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), - cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), - cpu_after: client_query(cpu_avg_query, time: timeframe_end) - } + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), + cpu_after: client_query(cpu_avg_query, time: timeframe_end) + } + end + end + end end end end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index 01d756d7284..66f29d95177 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -1,20 +1,25 @@ -module Gitlab::Prometheus::Queries - class EnvironmentQuery < BaseQuery - def query(environment_id) - environment = Environment.find_by(id: environment_id) - environment_slug = environment.slug - timeframe_start = 8.hours.ago.to_f - timeframe_end = Time.now.to_f +module Gitlab + module Prometheus + module Queries + class EnvironmentQuery < BaseQuery + def query(environment_id) + Environment.find_by(id: environment_id).try do |environment| + environment_slug = environment.slug + timeframe_start = 8.hours.ago.to_f + timeframe_end = Time.now.to_f - memory_query = raw_memory_usage_query(environment_slug) - cpu_query = raw_cpu_usage_query(environment_slug) + memory_query = raw_memory_usage_query(environment_slug) + cpu_query = raw_cpu_usage_query(environment_slug) - { - memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), - memory_current: client_query(memory_query, time: timeframe_end), - cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), - cpu_current: client_query(cpu_query, time: timeframe_end) - } + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_current: client_query(memory_query, time: timeframe_end), + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_current: client_query(cpu_query, time: timeframe_end) + } + end + end + end end end end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb new file mode 100644 index 00000000000..d4894c87f8d --- /dev/null +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -0,0 +1,80 @@ +module Gitlab + module Prometheus + module Queries + class MatchedMetricsQuery < BaseQuery + MAX_QUERY_ITEMS = 40.freeze + + def query + groups_data.map do |group, data| + { + group: group.name, + priority: group.priority, + active_metrics: data[:active_metrics], + metrics_missing_requirements: data[:metrics_missing_requirements] + } + end + end + + private + + def groups_data + metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all) + lookup = active_series_lookup(metrics_groups) + + groups = {} + + metrics_groups.each do |group| + groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } + active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) } + + groups[group][:active_metrics] += active_metrics + groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics + end + + groups + end + + def active_series_lookup(metric_groups) + timeframe_start = 8.hours.ago + timeframe_end = Time.now + + series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq + + lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| + client_series(*batched_series, start: timeframe_start, stop: timeframe_end) + .select(&method(:has_matching_label)) + .map { |series_info| [series_info['__name__'], true] } + end + lookup.to_h + end + + def has_matching_label(series_info) + series_info.key?('environment') + end + + def available_metrics + @available_metrics ||= client_label_values || [] + end + + def filter_active_metrics(metric_group) + metric_group.metrics.select! do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + metric_group + end + + def groups_with_active_metrics(metric_groups) + metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? } + end + + def metrics_with_required_series(metric_groups) + metric_groups.flat_map do |group| + group.metrics.select do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + end + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb new file mode 100644 index 00000000000..e44be770544 --- /dev/null +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -0,0 +1,73 @@ +module Gitlab + module Prometheus + module Queries + module QueryAdditionalMetrics + def query_metrics(query_context) + query_processor = method(:process_query).curry[query_context] + + groups = matched_metrics.map do |group| + metrics = group.metrics.map do |metric| + { + title: metric.title, + weight: metric.weight, + y_label: metric.y_label, + queries: metric.queries.map(&query_processor).select(&method(:query_with_result)) + } + end + + { + group: group.name, + priority: group.priority, + metrics: metrics.select(&method(:metric_with_any_queries)) + } + end + + groups.select(&method(:group_with_any_metrics)) + end + + private + + def metric_with_any_queries(metric) + metric[:queries]&.count&.> 0 + end + + def group_with_any_metrics(group) + group[:metrics]&.count&.> 0 + end + + def query_with_result(query) + query[:result]&.any? do |item| + item&.[](:values)&.any? || item&.[](:value)&.any? + end + end + + def process_query(context, query) + query_with_result = query.dup + result = + if query.key?(:query_range) + client_query_range(query[:query_range] % context, start: context[:timeframe_start], stop: context[:timeframe_end]) + else + client_query(query[:query] % context, time: context[:timeframe_end]) + end + query_with_result[:result] = result&.map(&:deep_symbolize_keys) + query_with_result + end + + def available_metrics + @available_metrics ||= client_label_values || [] + end + + def matched_metrics + result = Gitlab::Prometheus::MetricGroup.all.map do |group| + group.metrics.select! do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + group + end + + result.select { |group| group.metrics.any? } + end + end + end + end +end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 5b51a1779dd..aa94614bf18 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -29,6 +29,14 @@ module Gitlab end end + def label_values(name = '__name__') + json_api_get("label/#{name}/values") + end + + def series(*matches, start: 8.hours.ago, stop: Time.now) + json_api_get('series', 'match': matches, start: start.to_f, end: stop.to_f) + end + private def json_api_get(type, args = {}) diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index b1d6ea665b7..22554236c38 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -30,8 +30,8 @@ module Gitlab end def version_required - @version_required ||= File.read(Rails.root. - join('GITLAB_SHELL_VERSION')).strip + @version_required ||= File.read(Rails.root + .join('GITLAB_SHELL_VERSION')).strip end def strip_key(key) diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb index aa1468bff6b..b5f9d040047 100644 --- a/lib/gitlab/sherlock/line_profiler.rb +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -77,8 +77,8 @@ module Gitlab line_samples << LineSample.new(duration, events) end - samples << FileSample. - new(file, line_samples, total_duration, total_events) + samples << FileSample + .new(file, line_samples, total_duration, total_events) end samples diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 99e56e923eb..948bf5e6528 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -105,10 +105,10 @@ module Gitlab end def format_sql(query) - query.each_line. - map { |line| line.strip }. - join("\n"). - gsub(PREFIX_NEWLINE) { "\n#{$1} " } + query.each_line + .map { |line| line.strip } + .join("\n") + .gsub(PREFIX_NEWLINE) { "\n#{$1} " } end end end diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb index 5b1b03820a3..16ec002f139 100644 --- a/lib/gitlab/sql/recursive_cte.rb +++ b/lib/gitlab/sql/recursive_cte.rb @@ -52,10 +52,10 @@ module Gitlab # Applies the CTE to the given relation, returning a new one that will # query from it. def apply_to(relation) - relation.except(:where). - with. - recursive(to_arel). - from(alias_to(relation.model.arel_table)) + relation.except(:where) + .with + .recursive(to_arel) + .from(alias_to(relation.model.arel_table)) end end end |