diff options
Diffstat (limited to 'lib')
32 files changed, 332 insertions, 337 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 10b4f8934d7..2d6dd18d4ea 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -2,6 +2,19 @@ module API module Entities + class BlameRangeCommit < Grape::Entity + expose :id + expose :parent_ids + expose :message + expose :authored_date, :author_name, :author_email + expose :committed_date, :committer_name, :committer_email + end + + class BlameRange < Grape::Entity + expose :commit, using: BlameRangeCommit + expose :lines + end + class WikiPageBasic < Grape::Entity expose :format expose :slug @@ -366,10 +379,7 @@ module API end expose :request_access_enabled expose :full_name, :full_path - - if ::Group.supports_nested_objects? - expose :parent_id - end + expose :parent_id expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes diff --git a/lib/api/files.rb b/lib/api/files.rb index ca59d330e1c..0b438fb5bbc 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -83,6 +83,31 @@ module API resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do allow_access_with_scope :read_repository, if: -> (request) { request.get? || request.head? } + desc 'Get blame file metadata from repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + end + head ":id/repository/files/:file_path/blame", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + end + + desc 'Get blame file from the repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + end + get ":id/repository/files/:file_path/blame", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + + blame_ranges = Gitlab::Blame.new(@blob, @commit).groups(highlight: false) + present blame_ranges, with: Entities::BlameRange + end + desc 'Get raw file metadata from repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' diff --git a/lib/api/groups.rb b/lib/api/groups.rb index ec1020c7c78..f545f33c06b 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -114,10 +114,7 @@ module API params do requires :name, type: String, desc: 'The name of the group' requires :path, type: String, desc: 'The path of the group' - - if ::Group.supports_nested_objects? - optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' - end + optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' use :optional_params end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 8ae42c6dadd..1aa6dc44bf7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -544,5 +544,9 @@ module API params[:archived] end + + def ip_address + env["action_dispatch.remote_ip"].to_s || request.ip + end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 100463fcb95..5b87eccf860 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -25,7 +25,7 @@ module API end def get_runner_ip - { ip_address: env["action_dispatch.remote_ip"].to_s || request.ip } + { ip_address: ip_address } end def current_runner diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index e7fed55170e..b35aa952f81 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -27,7 +27,7 @@ module API requirements: { ref_name: /.+/ } do authorize_download_artifacts! - latest_build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) + latest_build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) present_carrierwave_file!(latest_build.artifacts_file) end @@ -45,7 +45,7 @@ module API requirements: { ref_name: /.+/ } do authorize_download_artifacts! - build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) + build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) path = Gitlab::Ci::Build::Artifacts::Path .new(params[:artifact_path]) diff --git a/lib/gitlab.rb b/lib/gitlab.rb index c62d1071dba..d9d8dcf7900 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_dependency File.expand_path('gitlab/popen', __dir__) +require 'pathname' module Gitlab def self.root @@ -61,7 +61,7 @@ module Gitlab def self.ee? @is_ee ||= - if ENV['IS_GITLAB_EE'].present? + if ENV['IS_GITLAB_EE'] && !ENV['IS_GITLAB_EE'].empty? Gitlab::Utils.to_boolean(ENV['IS_GITLAB_EE']) else # We may use this method when the Rails environment is not loaded. This diff --git a/lib/gitlab/background_migration/migrate_stage_index.rb b/lib/gitlab/background_migration/migrate_stage_index.rb index f921233460d..55608529cee 100644 --- a/lib/gitlab/background_migration/migrate_stage_index.rb +++ b/lib/gitlab/background_migration/migrate_stage_index.rb @@ -13,34 +13,22 @@ module Gitlab private def migrate_stage_index_sql(start_id, stop_id) - if Gitlab::Database.postgresql? - <<~SQL - WITH freqs AS ( - SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds - WHERE stage_id BETWEEN #{start_id} AND #{stop_id} - AND stage_idx IS NOT NULL - GROUP BY stage_id, stage_idx - ), indexes AS ( - SELECT DISTINCT stage_id, first_value(stage_idx) - OVER (PARTITION BY stage_id ORDER BY freq DESC) AS index - FROM freqs - ) + <<~SQL + WITH freqs AS ( + SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds + WHERE stage_id BETWEEN #{start_id} AND #{stop_id} + AND stage_idx IS NOT NULL + GROUP BY stage_id, stage_idx + ), indexes AS ( + SELECT DISTINCT stage_id, first_value(stage_idx) + OVER (PARTITION BY stage_id ORDER BY freq DESC) AS index + FROM freqs + ) - UPDATE ci_stages SET position = indexes.index - FROM indexes WHERE indexes.stage_id = ci_stages.id - AND ci_stages.position IS NULL; - SQL - else - <<~SQL - UPDATE ci_stages - SET position = - (SELECT stage_idx FROM ci_builds - WHERE ci_builds.stage_id = ci_stages.id - GROUP BY ci_builds.stage_idx ORDER BY COUNT(*) DESC LIMIT 1) - WHERE ci_stages.id BETWEEN #{start_id} AND #{stop_id} - AND ci_stages.position IS NULL - SQL - end + UPDATE ci_stages SET position = indexes.index + FROM indexes WHERE indexes.stage_id = ci_stages.id + AND ci_stages.position IS NULL; + SQL end end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 806c3a7a369..2ac51dd7b55 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -147,19 +147,13 @@ module Gitlab "#{UntrackedFile.table_name} (path) VALUES #{values}" end - def postgresql? - strong_memoize(:postgresql) do - Gitlab::Database.postgresql? - end - end - def can_bulk_insert_and_ignore_duplicates? !postgresql_pre_9_5? end def postgresql_pre_9_5? strong_memoize(:postgresql_pre_9_5) do - postgresql? && Gitlab::Database.version.to_f < 9.5 + Gitlab::Database.version.to_f < 9.5 end end diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb index 47579d46c1b..9ef6d8654ae 100644 --- a/lib/gitlab/background_migration/remove_restricted_todos.rb +++ b/lib/gitlab/background_migration/remove_restricted_todos.rb @@ -50,14 +50,7 @@ module Gitlab private def remove_non_members_todos(project_id) - if Gitlab::Database.postgresql? - batch_remove_todos_cte(project_id) - else - unauthorized_project_todos(project_id) - .each_batch(of: 5000) do |batch| - batch.delete_all - end - end + batch_remove_todos_cte(project_id) end def remove_confidential_issue_todos(project_id) @@ -90,13 +83,7 @@ module Gitlab next if target_types.empty? - if Gitlab::Database.postgresql? - batch_remove_todos_cte(project_id, target_types) - else - unauthorized_project_todos(project_id) - .where(target_type: target_types) - .delete_all - end + batch_remove_todos_cte(project_id, target_types) end end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 7f7cc62c8ef..15cccc6f287 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -14,7 +14,7 @@ module Gitlab @ref = ref @job = job - @pipeline = @project.ci_pipelines.latest_successful_for(@ref) + @pipeline = @project.ci_pipelines.latest_successful_for_ref(@ref) end def entity diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 7cabaadb122..3fbfdffe277 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -21,16 +21,10 @@ module Gitlab module MonthlyInterval # rubocop: disable CodeReuse/ActiveRecord def grouped_count(query) - if Gitlab::Database.postgresql? - query - .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") - .count(:created_at) - .transform_keys(&:squish) - else - query - .group("DATE_FORMAT(#{::Ci::Pipeline.table_name}.created_at, '01 %M %Y')") - .count(:created_at) - end + query + .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") + .count(:created_at) + .transform_keys(&:squish) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index d01b09f1398..df572188194 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -10,7 +10,7 @@ module Gitlab image: 'illustrations/manual_action.svg', size: 'svg-394', title: _('This job requires a manual action'), - content: _('This job depends on a user to trigger its process. Often they are used to deploy code to production environments') + content: _('This job requires manual intervention to start. Before starting this job, you can add variables below for last-minute configuration changes.') } end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index f7d046600e8..5b0b91de5da 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -84,11 +84,7 @@ module Gitlab .and(t[:created_at].lteq(Date.current.end_of_day)) .and(t[:author_id].eq(contributor.id)) - date_interval = if Gitlab::Database.postgresql? - "INTERVAL '#{Time.zone.now.utc_offset} seconds'" - else - "INTERVAL #{Time.zone.now.utc_offset} SECOND" - end + date_interval = "INTERVAL '#{Time.zone.now.utc_offset} seconds'" Event.reorder(nil) .select(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval}) AS date", 'count(id) as total_amount') diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index eef63536de4..cbdff0ab060 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -46,6 +46,7 @@ module Gitlab end end + # @deprecated def self.postgresql? adapter_name.casecmp('postgresql').zero? end @@ -79,19 +80,19 @@ module Gitlab end def self.postgresql_9_or_less? - postgresql? && version.to_f < 10 + version.to_f < 10 end def self.join_lateral_supported? - postgresql? && version.to_f >= 9.3 + version.to_f >= 9.3 end def self.replication_slots_supported? - postgresql? && version.to_f >= 9.4 + version.to_f >= 9.4 end def self.postgresql_minimum_supported_version? - postgresql? && version.to_f >= 9.6 + version.to_f >= 9.6 end # map some of the function names that changed between PostgreSQL 9 and 10 diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index 26adf4e221b..1f47f320a29 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -6,47 +6,25 @@ module Gitlab class Grant < ActiveRecord::Base include FromUnion - self.table_name = - if Database.postgresql? - 'information_schema.role_table_grants' - else - 'information_schema.schema_privileges' - end + self.table_name = 'information_schema.role_table_grants' # Returns true if the current user can create and execute triggers on the # given table. def self.create_and_execute_trigger?(table) - if Database.postgresql? - # We _must not_ use quote_table_name as this will produce double - # quotes on PostgreSQL and for "has_table_privilege" we need single - # quotes. - quoted_table = connection.quote(table) - - begin - from(nil) - .pluck(Arel.sql("has_table_privilege(#{quoted_table}, 'TRIGGER')")) - .first - rescue ActiveRecord::StatementInvalid - # This error is raised when using a non-existing table name. In this - # case we just want to return false as a user technically can't - # create triggers for such a table. - false - end - else - queries = [ - Grant.select(1) - .from('information_schema.user_privileges') - .where("PRIVILEGE_TYPE = 'SUPER'") - .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"), - - Grant.select(1) - .from('information_schema.schema_privileges') - .where("PRIVILEGE_TYPE = 'TRIGGER'") - .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name) - .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") - ] + # We _must not_ use quote_table_name as this will produce double + # quotes on PostgreSQL and for "has_table_privilege" we need single + # quotes. + quoted_table = connection.quote(table) - Grant.from_union(queries, alias_as: 'privs').any? + begin + from(nil) + .pluck(Arel.sql("has_table_privilege(#{quoted_table}, 'TRIGGER')")) + .first + rescue ActiveRecord::StatementInvalid + # This error is raised when using a non-existing table name. In this + # case we just want to return false as a user technically can't + # create triggers for such a table. + false end end end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 391c1e85a7d..603b125d8b4 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -137,8 +137,6 @@ module Gitlab end def extract_diff_epoch(diff) - return diff unless Gitlab::Database.postgresql? - Arel.sql(%Q{EXTRACT(EPOCH FROM (#{diff.to_sql}))}) end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 4bd09163bf2..9bba4f6ce1e 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -152,8 +152,6 @@ module Gitlab # Only available on Postgresql >= 9.2 def supports_drop_index_concurrently? - return false unless Database.postgresql? - version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i version >= 90200 diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb index 109ae7893da..ddbabc9098e 100644 --- a/lib/gitlab/database/sha_attribute.rb +++ b/lib/gitlab/database/sha_attribute.rb @@ -2,14 +2,9 @@ module Gitlab module Database - BINARY_TYPE = - if Gitlab::Database.postgresql? - # PostgreSQL defines its own class with slightly different - # behaviour from the default Binary type. - ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea - else - ActiveModel::Type::Binary - end + # PostgreSQL defines its own class with slightly different + # behaviour from the default Binary type. + BINARY_TYPE = ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa). # diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb index 2bcd25cd3cc..ec4b97eaca4 100644 --- a/lib/gitlab/health_checks/db_check.rb +++ b/lib/gitlab/health_checks/db_check.rb @@ -18,11 +18,7 @@ module Gitlab def check catch_timeout 10.seconds do - if Gitlab::Database.postgresql? - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s - else - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s - end + ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s end end end diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 38b32770e90..c06f106ffe1 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -32,11 +32,6 @@ module Gitlab # Returns the maximum depth starting from the base # A base object with no children has a maximum depth of `1` def max_descendants_depth - unless hierarchy_supported? - # This makes the return value consistent with the case where hierarchy is supported - return descendants_base.exists? ? 1 : nil - end - base_and_descendants(with_depth: true).maximum(DEPTH_COLUMN) end @@ -66,8 +61,6 @@ module Gitlab # each parent. # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors(upto: nil, hierarchy_order: nil) - return ancestors_base unless hierarchy_supported? - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order @@ -81,10 +74,6 @@ module Gitlab # When `with_depth` is `true`, a `depth` column is included where it starts with `1` for the base objects # and incremented as we go down the descendant tree def base_and_descendants(with_depth: false) - unless hierarchy_supported? - return with_depth ? descendants_base.select("1 as #{DEPTH_COLUMN}", objects_table[Arel.star]) : descendants_base - end - read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all)) end @@ -112,8 +101,6 @@ module Gitlab # If nested objects are not supported, ancestors_base is returned. # rubocop: disable CodeReuse/ActiveRecord def all_objects - return ancestors_base unless hierarchy_supported? - ancestors = base_and_ancestors_cte descendants = base_and_descendants_cte @@ -135,10 +122,6 @@ module Gitlab private - def hierarchy_supported? - Gitlab::Database.postgresql? - end - # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors_cte(stop_id = nil, hierarchy_order = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index a13b3f9e069..f96466b2b00 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -175,6 +175,10 @@ module Gitlab @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze end + def project_wiki_git_route_regex + @project_wiki_git_route_regex ||= /#{PATH_REGEX_STR}\.wiki/.freeze + end + def full_namespace_path_regex @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z} end diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb new file mode 100644 index 00000000000..a9270cd536e --- /dev/null +++ b/lib/gitlab/project_authorizations.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +# This class relies on Common Table Expressions to efficiently get all data, +# including data for nested groups. +module Gitlab + class ProjectAuthorizations + attr_reader :user + + # user - The User object for which to calculate the authorizations. + def initialize(user) + @user = user + end + + def calculate + cte = recursive_cte + cte_alias = cte.table.alias(Group.table_name) + projects = Project.arel_table + links = ProjectGroupLink.arel_table + + relations = [ + # The project a user has direct access to. + user.projects.select_for_project_authorization, + + # The personal projects of the user. + user.personal_projects.select_as_maintainer_for_project_authorization, + + # Projects that belong directly to any of the groups the user has + # access to. + Namespace + .unscoped + .select([alias_as_column(projects[:id], 'project_id'), + cte_alias[:access_level]]) + .from(cte_alias) + .joins(:projects), + + # Projects shared with any of the namespaces the user has access to. + Namespace + .unscoped + .select([ + links[:project_id], + least(cte_alias[:access_level], links[:group_access], 'access_level') + ]) + .from(cte_alias) + .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') + .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') + .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') + .where('p_ns.share_with_group_lock IS FALSE') + ] + + ProjectAuthorization + .unscoped + .with + .recursive(cte.to_arel) + .select_from_union(relations) + end + + private + + # Builds a recursive CTE that gets all the groups the current user has + # access to, including any nested groups. + def recursive_cte + cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) + members = Member.arel_table + namespaces = Namespace.arel_table + + # Namespaces the user is a member of. + cte << user.groups + .select([namespaces[:id], members[:access_level]]) + .except(:order) + + # Sub groups of any groups the user is a member of. + cte << Group.select([ + namespaces[:id], + greatest(members[:access_level], cte.table[:access_level], 'access_level') + ]) + .joins(join_cte(cte)) + .joins(join_members) + .except(:order) + + cte + end + + # Builds a LEFT JOIN to join optional memberships onto the CTE. + def join_members + members = Member.arel_table + namespaces = Namespace.arel_table + + cond = members[:source_id] + .eq(namespaces[:id]) + .and(members[:source_type].eq('Namespace')) + .and(members[:requested_at].eq(nil)) + .and(members[:user_id].eq(user.id)) + + Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) + end + + # Builds an INNER JOIN to join namespaces onto the CTE. + def join_cte(cte) + namespaces = Namespace.arel_table + cond = cte.table[:id].eq(namespaces[:parent_id]) + + Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) + end + + def greatest(left, right, column_alias) + sql_function('GREATEST', [left, right], column_alias) + end + + def least(left, right, column_alias) + sql_function('LEAST', [left, right], column_alias) + end + + def sql_function(name, args, column_alias) + alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) + end + + def alias_as_column(value, alias_to) + Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) + end + end +end diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb deleted file mode 100644 index 2372a316ab0..00000000000 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ /dev/null @@ -1,125 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ProjectAuthorizations - # Calculating new project authorizations when supporting nested groups. - # - # This class relies on Common Table Expressions to efficiently get all data, - # including data for nested groups. As a result this class can only be used - # on PostgreSQL. - class WithNestedGroups - attr_reader :user - - # user - The User object for which to calculate the authorizations. - def initialize(user) - @user = user - end - - def calculate - cte = recursive_cte - cte_alias = cte.table.alias(Group.table_name) - projects = Project.arel_table - links = ProjectGroupLink.arel_table - - relations = [ - # The project a user has direct access to. - user.projects.select_for_project_authorization, - - # The personal projects of the user. - user.personal_projects.select_as_maintainer_for_project_authorization, - - # Projects that belong directly to any of the groups the user has - # access to. - Namespace - .unscoped - .select([alias_as_column(projects[:id], 'project_id'), - cte_alias[:access_level]]) - .from(cte_alias) - .joins(:projects), - - # Projects shared with any of the namespaces the user has access to. - Namespace - .unscoped - .select([links[:project_id], - least(cte_alias[:access_level], - links[:group_access], - 'access_level')]) - .from(cte_alias) - .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') - .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') - .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') - .where('p_ns.share_with_group_lock IS FALSE') - ] - - ProjectAuthorization - .unscoped - .with - .recursive(cte.to_arel) - .select_from_union(relations) - end - - private - - # Builds a recursive CTE that gets all the groups the current user has - # access to, including any nested groups. - def recursive_cte - cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) - members = Member.arel_table - namespaces = Namespace.arel_table - - # Namespaces the user is a member of. - cte << user.groups - .select([namespaces[:id], members[:access_level]]) - .except(:order) - - # Sub groups of any groups the user is a member of. - cte << Group.select([namespaces[:id], - greatest(members[:access_level], - cte.table[:access_level], 'access_level')]) - .joins(join_cte(cte)) - .joins(join_members) - .except(:order) - - cte - end - - # Builds a LEFT JOIN to join optional memberships onto the CTE. - def join_members - members = Member.arel_table - namespaces = Namespace.arel_table - - cond = members[:source_id] - .eq(namespaces[:id]) - .and(members[:source_type].eq('Namespace')) - .and(members[:requested_at].eq(nil)) - .and(members[:user_id].eq(user.id)) - - Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) - end - - # Builds an INNER JOIN to join namespaces onto the CTE. - def join_cte(cte) - namespaces = Namespace.arel_table - cond = cte.table[:id].eq(namespaces[:parent_id]) - - Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) - end - - def greatest(left, right, column_alias) - sql_function('GREATEST', [left, right], column_alias) - end - - def least(left, right, column_alias) - sql_function('LEAST', [left, right], column_alias) - end - - def sql_function(name, args, column_alias) - alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) - end - - def alias_as_column(value, alias_to) - Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) - end - end - end -end diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb deleted file mode 100644 index 50b41b17649..00000000000 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ProjectAuthorizations - # Calculating new project authorizations when not supporting nested groups. - class WithoutNestedGroups - attr_reader :user - - # user - The User object for which to calculate the authorizations. - def initialize(user) - @user = user - end - - def calculate - relations = [ - # Projects the user is a direct member of - user.projects.select_for_project_authorization, - - # Personal projects - user.personal_projects.select_as_maintainer_for_project_authorization, - - # Projects of groups the user is a member of - user.groups_projects.select_for_project_authorization, - - # Projects shared with groups the user is a member of - user.groups.joins(:shared_projects).select_for_project_authorization - ] - - ProjectAuthorization - .unscoped - .select_from_union(relations) - end - end - end -end diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 159ce27e702..cbd89b7629f 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -96,12 +96,7 @@ module Gitlab private def raw_explain(query) - explain = - if Gitlab::Database.postgresql? - "EXPLAIN ANALYZE #{query};" - else - "EXPLAIN #{query};" - end + explain = "EXPLAIN ANALYZE #{query};" ActiveRecord::Base.connection.execute(explain) end diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb new file mode 100644 index 00000000000..b06ffa9c121 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/metrics.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class Metrics + def initialize + @metrics = init_metrics + end + + def call(_worker, job, queue) + labels = create_labels(queue) + @metrics[:sidekiq_running_jobs].increment(labels, 1) + + if job['retry_count'].present? + @metrics[:sidekiq_jobs_retried_total].increment(labels, 1) + end + + realtime = Benchmark.realtime do + yield + end + + @metrics[:sidekiq_jobs_completion_seconds].observe(labels, realtime) + rescue Exception # rubocop: disable Lint/RescueException + @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) + raise + ensure + @metrics[:sidekiq_running_jobs].increment(labels, -1) + end + + private + + def init_metrics + { + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete sidekiq job'), + sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), + sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :livesum) + } + end + + def create_labels(queue) + { + queue: queue + } + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index db1086c9cae..d5657c474c8 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -137,8 +137,11 @@ module Gitlab # @return [Array<#totals>] An array of objects that respond to `#totals` def usage_data_counters - [Gitlab::UsageDataCounters::WikiPageCounter, - Gitlab::UsageDataCounters::WebIdeCounter] + [ + Gitlab::UsageDataCounters::WikiPageCounter, + Gitlab::UsageDataCounters::WebIdeCounter, + Gitlab::UsageDataCounters::SearchCounter + ] end def components_usage_data diff --git a/lib/gitlab/usage_data_counters/search_counter.rb b/lib/gitlab/usage_data_counters/search_counter.rb new file mode 100644 index 00000000000..5f0735347e1 --- /dev/null +++ b/lib/gitlab/usage_data_counters/search_counter.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class SearchCounter + extend RedisCounter + + NAVBAR_SEARCHES_COUNT_KEY = 'NAVBAR_SEARCHES_COUNT' + + class << self + def increment_navbar_searches_count + increment(NAVBAR_SEARCHES_COUNT_KEY) + end + + def total_navbar_searches_count + total_count(NAVBAR_SEARCHES_COUNT_KEY) + end + + def totals + { + navbar_searches: total_navbar_searches_count + } + end + end + end + end +end diff --git a/lib/prometheus/pid_provider.rb b/lib/prometheus/pid_provider.rb index c92522c73c5..e0f7e7e0a9e 100644 --- a/lib/prometheus/pid_provider.rb +++ b/lib/prometheus/pid_provider.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'prometheus/client/support/unicorn' - module Prometheus module PidProvider extend self @@ -10,29 +8,38 @@ module Prometheus if Sidekiq.server? 'sidekiq' elsif defined?(Unicorn::Worker) - "unicorn_#{unicorn_worker_id}" + unicorn_worker_id elsif defined?(::Puma) - "puma_#{puma_worker_id}" + puma_worker_id else - "process_#{Process.pid}" + unknown_process_id end end private - # This is not fully accurate as we don't really know if the nil returned - # is actually means we're on master or not. - # Follow up issue was created to address this problem and - # to introduce more structrured approach to a current process discovery: - # https://gitlab.com/gitlab-org/gitlab-ce/issues/64740 def unicorn_worker_id - ::Prometheus::Client::Support::Unicorn.worker_id || 'master' + if matches = process_name.match(/unicorn.*worker\[([0-9]+)\]/) + "unicorn_#{matches[1]}" + elsif process_name =~ /unicorn/ + "unicorn_master" + else + unknown_process_id + end end - # See the comment for #unicorn_worker_id def puma_worker_id - match = process_name.match(/cluster worker ([0-9]+):/) - match ? match[1] : 'master' + if matches = process_name.match(/puma.*cluster worker ([0-9]+):/) + "puma_#{matches[1]}" + elsif process_name =~ /puma/ + "puma_master" + else + unknown_process_id + end + end + + def unknown_process_id + "process_#{Process.pid}" end def process_name diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index d054959e05e..1961f64659c 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -70,5 +70,13 @@ namespace :gitlab do Gitlab::DowntimeCheck.new.check_and_print(migrations) end + + desc 'Sets up EE specific database functionality' + + if Gitlab.ee? + task setup_ee: %w[geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate] + else + task :setup_ee + end end end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index e763de682f8..5d86d6e466c 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -45,8 +45,6 @@ namespace :gitlab do # method terminates all the connections so that a subsequent DROP # will work. def self.terminate_all_connections - return false unless Gitlab::Database.postgresql? - cmd = <<~SQL SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity |