diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-20 18:42:06 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-20 18:42:06 +0000 |
| commit | 6e4e1050d9dba2b7b2523fdd1768823ab85feef4 (patch) | |
| tree | 78be5963ec075d80116a932011d695dd33910b4e /lib/gitlab | |
| parent | 1ce776de4ae122aba3f349c02c17cebeaa8ecf07 (diff) | |
| download | gitlab-ce-6e4e1050d9dba2b7b2523fdd1768823ab85feef4.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-3-stable-ee
Diffstat (limited to 'lib/gitlab')
293 files changed, 4789 insertions, 2689 deletions
diff --git a/lib/gitlab/alerting/alert.rb b/lib/gitlab/alerting/alert.rb index dad3dabb4fc..94b81b7d290 100644 --- a/lib/gitlab/alerting/alert.rb +++ b/lib/gitlab/alerting/alert.rb @@ -7,7 +7,17 @@ module Gitlab include Gitlab::Utils::StrongMemoize include Presentable - attr_accessor :project, :payload + attr_accessor :project, :payload, :am_alert + + def self.for_alert_management_alert(project:, alert:) + params = if alert.prometheus? + alert.payload + else + Gitlab::Alerting::NotificationPayloadParser.call(alert.payload.to_h, alert.project) + end + + self.new(project: project, payload: params, am_alert: alert) + end def gitlab_alert strong_memoize(:gitlab_alert) do diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index 4d47a17545a..be5d9be3d64 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -13,7 +13,7 @@ module Gitlab MAPPINGS = { Issue => { serializer_class: AnalyticsIssueSerializer, - includes_for_query: { project: [:namespace], author: [] }, + includes_for_query: { project: { namespace: [:route] }, author: [] }, columns_for_select: %I[title iid id created_at author_id project_id] }, MergeRequest => { @@ -41,7 +41,7 @@ module Gitlab project = record.project attributes = record.attributes.merge({ project_path: project.path, - namespace_path: project.namespace.path, + namespace_path: project.namespace.route.path, author: record.author }) serializer.represent(attributes) @@ -82,7 +82,7 @@ module Gitlab q = ordered_and_limited_query .joins(ci_build_join) - .select(build_table[:id], round_duration_to_seconds.as('total_time')) + .select(build_table[:id], *time_columns) results = execute_query(q).to_a @@ -90,12 +90,12 @@ module Gitlab end def ordered_and_limited_query - order_by_end_event(query).limit(MAX_RECORDS) + order_by_end_event(query, columns).limit(MAX_RECORDS) end def records results = ordered_and_limited_query - .select(*columns, round_duration_to_seconds.as('total_time')) + .select(*columns, *time_columns) # using preloader instead of includes to avoid AR generating a large column list ActiveRecord::Associations::Preloader.new.preload( @@ -106,6 +106,14 @@ module Gitlab results end # rubocop: enable CodeReuse/ActiveRecord + + def time_columns + [ + stage.start_event.timestamp_projection.as('start_event_timestamp'), + stage.end_event.timestamp_projection.as('end_event_timestamp'), + round_duration_to_seconds.as('total_time') + ] + end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb index c9a75b39959..80e426e6e17 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb @@ -24,13 +24,13 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def order_by_end_event(query) + def order_by_end_event(query, extra_columns_to_select = [:id]) ordered_query = query.reorder(stage.end_event.timestamp_projection.desc) # When filtering for more than one label, postgres requires the columns in ORDER BY to be present in the GROUP BY clause if requires_grouping? column_list = [ - ordered_query.arel_table[:id], + *extra_columns_to_select, *stage.end_event.column_list, *stage.start_event.column_list ] diff --git a/lib/gitlab/analytics/unique_visits.rb b/lib/gitlab/analytics/unique_visits.rb index 9dd7d048eec..ad746ebbd42 100644 --- a/lib/gitlab/analytics/unique_visits.rb +++ b/lib/gitlab/analytics/unique_visits.rb @@ -3,57 +3,36 @@ module Gitlab module Analytics class UniqueVisits - TARGET_IDS = Set[ - 'g_analytics_contribution', - 'g_analytics_insights', - 'g_analytics_issues', - 'g_analytics_productivity', - 'g_analytics_valuestream', - 'p_analytics_pipelines', - 'p_analytics_code_reviews', - 'p_analytics_valuestream', - 'p_analytics_insights', - 'p_analytics_issues', - 'p_analytics_repo', - 'u_analytics_todos', - 'i_analytics_cohorts', - 'i_analytics_dev_ops_score' - ].freeze - - KEY_EXPIRY_LENGTH = 28.days - def track_visit(visitor_id, target_id, time = Time.zone.now) - target_key = key(target_id, time) - - Gitlab::Redis::SharedState.with do |redis| - redis.multi do |multi| - multi.pfadd(target_key, visitor_id) - multi.expire(target_key, KEY_EXPIRY_LENGTH) - end - end + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(visitor_id, target_id, time) end - def weekly_unique_visits_for_target(target_id, week_of: 7.days.ago) - Gitlab::Redis::SharedState.with do |redis| - redis.pfcount(key(target_id, week_of)) - end + # Returns number of unique visitors for given targets in given time frame + # + # @param [String, Array[<String>]] targets ids of targets to count visits on. Special case for :any + # @param [ActiveSupport::TimeWithZone] start_date start of time frame + # @param [ActiveSupport::TimeWithZone] end_date end of time frame + # @return [Integer] number of unique visitors + def unique_visits_for(targets:, start_date: 7.days.ago, end_date: start_date + 1.week) + target_ids = if targets == :analytics + self.class.analytics_ids + elsif targets == :compliance + self.class.compliance_ids + else + Array(targets) + end + + Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: target_ids, start_date: start_date, end_date: end_date) end - def weekly_unique_visits_for_any_target(week_of: 7.days.ago) - keys = TARGET_IDS.map { |target_id| key(target_id, week_of) } - - Gitlab::Redis::SharedState.with do |redis| - redis.pfcount(*keys) + class << self + def analytics_ids + Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('analytics') end - end - - private - def key(target_id, time) - raise "Invalid target id #{target_id}" unless TARGET_IDS.include?(target_id.to_s) - - year_week = time.strftime('%G-%V') - "#{target_id}-{#{year_week}}" + def compliance_ids + Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('compliance') + end end end end diff --git a/lib/gitlab/app_logger.rb b/lib/gitlab/app_logger.rb index 3f5e9adf925..a39e7f31886 100644 --- a/lib/gitlab/app_logger.rb +++ b/lib/gitlab/app_logger.rb @@ -5,7 +5,11 @@ module Gitlab LOGGERS = [Gitlab::AppTextLogger, Gitlab::AppJsonLogger].freeze def self.loggers - LOGGERS + if Gitlab::Utils.to_boolean(ENV.fetch('UNSTRUCTURED_RAILS_LOG', 'true')) + LOGGERS + else + [Gitlab::AppJsonLogger] + end end def self.primary_logger diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 2fac76d03e8..5cacd7e5983 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -4,6 +4,7 @@ require 'asciidoctor' require 'asciidoctor-plantuml' require 'asciidoctor/extensions' require 'gitlab/asciidoc/html5_converter' +require 'gitlab/asciidoc/mermaid_block_processor' require 'gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter' module Gitlab @@ -46,6 +47,7 @@ module Gitlab def self.render(input, context) extensions = proc do include_processor ::Gitlab::Asciidoc::IncludeProcessor.new(context) + block ::Gitlab::Asciidoc::MermaidBlockProcessor end extra_attrs = path_attrs(context[:requested_path]) diff --git a/lib/gitlab/asciidoc/mermaid_block_processor.rb b/lib/gitlab/asciidoc/mermaid_block_processor.rb new file mode 100644 index 00000000000..03d1c7f1961 --- /dev/null +++ b/lib/gitlab/asciidoc/mermaid_block_processor.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'asciidoctor' + +module Gitlab + module Asciidoc + # Mermaid BlockProcessor + class MermaidBlockProcessor < ::Asciidoctor::Extensions::BlockProcessor + use_dsl + + named :mermaid + on_context :literal, :listing + parse_content_as :simple + + def process(parent, reader, attrs) + create_mermaid_source_block(parent, reader.read, attrs) + end + + private + + def create_mermaid_source_block(parent, content, attrs) + # If "subs" attribute is specified, substitute accordingly. + # Be careful not to specify "specialcharacters" or your diagram code won't be valid anymore! + subs = attrs['subs'] + content = parent.apply_subs(content, parent.resolve_subs(subs)) if subs + html = %(<div><pre data-mermaid-style="display">#{CGI.escape_html(content)}</pre></div>) + ::Asciidoctor::Block.new(parent, :pass, { + content_model: :raw, + source: html, + subs: :default + }.merge(attrs)) + end + end + end +end diff --git a/lib/gitlab/audit/deleted_author.rb b/lib/gitlab/audit/deleted_author.rb new file mode 100644 index 00000000000..e3b8ad5ad21 --- /dev/null +++ b/lib/gitlab/audit/deleted_author.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab + module Audit + class DeletedAuthor < Gitlab::Audit::NullAuthor + end + end +end diff --git a/lib/gitlab/audit/null_author.rb b/lib/gitlab/audit/null_author.rb new file mode 100644 index 00000000000..0b0e6a46fe4 --- /dev/null +++ b/lib/gitlab/audit/null_author.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Audit + class NullAuthor + attr_reader :id, :name + + # Creates an Author + # + # While tracking events that could take place even when + # a user is not logged in, (eg: downloading repo of a public project), + # we set the author_id of such events as -1 + # + # @param [Integer] id + # @param [String] name + # + # @return [Gitlab::Audit::UnauthenticatedAuthor, Gitlab::Audit::DeletedAuthor] + def self.for(id, name) + if id == -1 + Gitlab::Audit::UnauthenticatedAuthor.new(name: name) + else + Gitlab::Audit::DeletedAuthor.new(id: id, name: name) + end + end + + def initialize(id:, name:) + @id = id + @name = name + end + + def current_sign_in_ip + nil + end + end + end +end diff --git a/lib/gitlab/audit/unauthenticated_author.rb b/lib/gitlab/audit/unauthenticated_author.rb new file mode 100644 index 00000000000..84c323c1950 --- /dev/null +++ b/lib/gitlab/audit/unauthenticated_author.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Audit + class UnauthenticatedAuthor < Gitlab::Audit::NullAuthor + def initialize(name: nil) + super(id: -1, name: name) + end + + # Events that are authored by unauthenticated users, should be + # shown as authored by `An unauthenticated user` in the UI. + def name + @name || _('An unauthenticated user') + end + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 1a23814959d..332d0bc1478 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -26,6 +26,8 @@ module Gitlab # Default scopes for OAuth applications that don't define their own DEFAULT_SCOPES = [:api].freeze + CI_JOB_USER = 'gitlab-ci-token' + class << self prepend_if_ee('EE::Gitlab::Auth') # rubocop: disable Cop/InjectEnterpriseEditionModule @@ -126,7 +128,7 @@ module Gitlab # rubocop:enable Gitlab/RailsLogger def skip_rate_limit?(login:) - ::Ci::Build::CI_REGISTRY_USER == login + CI_JOB_USER == login end def look_to_limit_user(actor) @@ -257,7 +259,7 @@ module Gitlab end def build_access_token_check(login, password) - return unless login == 'gitlab-ci-token' + return unless login == CI_JOB_USER return unless password build = find_build_by_token(password) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index bd5aed0d964..f3d0c053880 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -20,6 +20,7 @@ module Gitlab module AuthFinders include Gitlab::Utils::StrongMemoize include ActionController::HttpAuthentication::Basic + include ActionController::HttpAuthentication::Token PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN' PRIVATE_TOKEN_PARAM = :private_token @@ -81,7 +82,7 @@ module Gitlab login, password = user_name_and_password(current_request) return unless login.present? && password.present? - return unless ::Ci::Build::CI_REGISTRY_USER == login + return unless ::Gitlab::Auth::CI_JOB_USER == login job = ::Ci::Build.find_by_token(password) raise UnauthorizedError unless job @@ -131,6 +132,15 @@ module Gitlab deploy_token end + def cluster_agent_token_from_authorization_token + return unless route_authentication_setting[:cluster_agent_token_allowed] + return unless current_request.authorization.present? + + authorization_token, _options = token_and_options(current_request) + + ::Clusters::AgentToken.find_by_token(authorization_token) + end + def find_runner_from_token return unless api_request? diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index f64fcd822c6..4f448211abf 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -54,7 +54,7 @@ module Gitlab if results.nil? response = ldap.get_operation_result - unless response.code.zero? + unless response.code == 0 Rails.logger.warn("LDAP search error: #{response.message}") # rubocop:disable Gitlab/RailsLogger end diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index b3321c0b1fb..8c5000147c4 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -11,7 +11,7 @@ module Gitlab InvalidEntryError = Class.new(StandardError) - attr_accessor :entry, :provider + attr_accessor :provider def self.find_by_uid(uid, adapter) uid = Net::LDAP::Filter.escape(uid) diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 8a60d6ef482..086f4a2e91c 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -12,7 +12,7 @@ module Gitlab SignupDisabledError = Class.new(StandardError) SigninDisabledForProviderError = Class.new(StandardError) - attr_accessor :auth_hash, :gl_user + attr_reader :auth_hash def initialize(auth_hash) self.auth_hash = auth_hash @@ -62,6 +62,7 @@ module Gitlab def find_user user = find_by_uid_and_provider + user ||= find_by_email if auto_link_user? user ||= find_or_build_ldap_user if auto_link_ldap_user? user ||= build_new_user if signup_enabled? @@ -150,6 +151,7 @@ module Gitlab def find_ldap_person(auth_hash, adapter) Gitlab::Auth::Ldap::Person.find_by_uid(auth_hash.uid, adapter) || Gitlab::Auth::Ldap::Person.find_by_email(auth_hash.uid, adapter) || + Gitlab::Auth::Ldap::Person.find_by_email(auth_hash.email, adapter) || Gitlab::Auth::Ldap::Person.find_by_dn(auth_hash.uid, adapter) rescue Gitlab::Auth::Ldap::LdapConnectionError nil @@ -269,6 +271,10 @@ module Gitlab .disabled_oauth_sign_in_sources .include?(auth_hash.provider) end + + def auto_link_user? + Gitlab.config.omniauth.auto_link_user + end end end end diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb deleted file mode 100644 index 79f38aed9f1..00000000000 --- a/lib/gitlab/background_migration/archive_legacy_traces.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class ArchiveLegacyTraces - def perform(start_id, stop_id) - # This background migration directly refers to ::Ci::Build model which is defined in application code. - # In general, migration code should be isolated as much as possible in order to be idempotent. - # However, `archive!` method is too complicated to be replicated by coping its subsequent code. - # So we chose a way to use ::Ci::Build directly and we don't change the `archive!` method until 11.1 - ::Ci::Build.finished.without_archived_trace - .where(id: start_id..stop_id).find_each do |build| - build.trace.archive! - rescue => e - Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger - end - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_designs_relative_position.rb b/lib/gitlab/background_migration/backfill_designs_relative_position.rb new file mode 100644 index 00000000000..efbb1b950ad --- /dev/null +++ b/lib/gitlab/background_migration/backfill_designs_relative_position.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration is not needed anymore and was disabled, because we're now + # also backfilling design positions immediately before moving a design. + # + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39555 + class BackfillDesignsRelativePosition + def perform(issue_ids) + # no-op + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb b/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb deleted file mode 100644 index a6194616663..00000000000 --- a/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class that will fill the project_repositories table for projects that - # are on hashed storage and an entry is is missing in this table. - class BackfillHashedProjectRepositories < BackfillProjectRepositories - private - - def projects - Project.on_hashed_storage - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb b/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb deleted file mode 100644 index 2a079060380..00000000000 --- a/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb +++ /dev/null @@ -1,213 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This module is used to write the full path of all projects to - # the git repository config file. - # Storing the full project path in the git config allows admins to - # easily identify a project when it is using hashed storage. - module BackfillProjectFullpathInRepoConfig - OrphanedNamespaceError = Class.new(StandardError) - - module Storage - # Class that returns the disk path for a project using hashed storage - class Hashed - attr_accessor :project - - ROOT_PATH_PREFIX = '@hashed' - - def initialize(project) - @project = project - end - - def disk_path - "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}" - end - - def disk_hash - @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id - end - end - - # Class that returns the disk path for a project using legacy storage - class LegacyProject - attr_accessor :project - - def initialize(project) - @project = project - end - - def disk_path - project.full_path - end - end - end - - # Concern used by Project and Namespace to determine the full - # route to the project - module Routable - extend ActiveSupport::Concern - - def full_path - @full_path ||= build_full_path - end - - def build_full_path - return path unless has_parent? - - raise OrphanedNamespaceError if parent.nil? - - parent.full_path + '/' + path - end - - def has_parent? - read_attribute(association(:parent).reflection.foreign_key) - end - end - - # Class used to interact with repository using Gitaly - class Repository - attr_reader :storage - - def initialize(storage, relative_path) - @storage = storage - @relative_path = relative_path - end - - def gitaly_repository - Gitaly::Repository.new(storage_name: @storage, relative_path: @relative_path) - end - end - - # Namespace can be a user or group. It can be the root or a - # child of another namespace. - class Namespace < ActiveRecord::Base - self.table_name = 'namespaces' - self.inheritance_column = nil - - include Routable - - belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces' - has_many :projects, inverse_of: :parent - has_many :namespaces, inverse_of: :parent - end - - # Project is where the repository (etc.) is stored - class Project < ActiveRecord::Base - self.table_name = 'projects' - - include Routable - include EachBatch - - FULLPATH_CONFIG_KEY = 'gitlab.fullpath' - - belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects' - delegate :disk_path, to: :storage - - def add_fullpath_config - entries = { FULLPATH_CONFIG_KEY => full_path } - - repository_service.set_config(entries) - end - - def remove_fullpath_config - repository_service.delete_config([FULLPATH_CONFIG_KEY]) - end - - def cleanup_repository - repository_service.cleanup - end - - def storage - @storage ||= - if hashed_storage? - Storage::Hashed.new(self) - else - Storage::LegacyProject.new(self) - end - end - - def hashed_storage? - self.storage_version && self.storage_version >= 1 - end - - def repository - @repository ||= Repository.new(repository_storage, disk_path + '.git') - end - - def repository_service - @repository_service ||= Gitlab::GitalyClient::RepositoryService.new(repository) - end - end - - # Base class for Up and Down migration classes - class BackfillFullpathMigration - RETRY_DELAY = 15.minutes - MAX_RETRIES = 2 - - # Base class for retrying one project - class BaseRetryOne - def perform(project_id, retry_count) - project = Project.find(project_id) - - return unless project - - migration_class.new.safe_perform_one(project, retry_count) - end - end - - def perform(start_id, end_id) - Project.includes(:parent).where(id: start_id..end_id).each do |project| - safe_perform_one(project) - end - end - - def safe_perform_one(project, retry_count = 0) - perform_one(project) - rescue GRPC::NotFound, GRPC::InvalidArgument, OrphanedNamespaceError - nil - rescue GRPC::BadStatus - schedule_retry(project, retry_count + 1) if retry_count < MAX_RETRIES - end - - def schedule_retry(project, retry_count) - # Constants provided to BackgroundMigrationWorker must be within the - # scope of Gitlab::BackgroundMigration - retry_class_name = self.class::RetryOne.name.sub('Gitlab::BackgroundMigration::', '') - - BackgroundMigrationWorker.perform_in(RETRY_DELAY, retry_class_name, [project.id, retry_count]) - end - end - - # Class to add the fullpath to the git repo config - class Up < BackfillFullpathMigration - # Class used to retry - class RetryOne < BaseRetryOne - def migration_class - Up - end - end - - def perform_one(project) - project.cleanup_repository - project.add_fullpath_config - end - end - - # Class to rollback adding the fullpath to the git repo config - class Down < BackfillFullpathMigration - # Class used to retry - class RetryOne < BaseRetryOne - def migration_class - Down - end - end - - def perform_one(project) - project.cleanup_repository - project.remove_fullpath_config - end - end - end - end -end diff --git a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb new file mode 100644 index 00000000000..6014ccc12eb --- /dev/null +++ b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class CopyMergeRequestTargetProjectToMergeRequestMetrics + extend ::Gitlab::Utils::Override + + def perform(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + WITH merge_requests_batch AS ( + SELECT id, target_project_id + FROM merge_requests WHERE id BETWEEN #{Integer(start_id)} AND #{Integer(stop_id)} + ) + UPDATE + merge_request_metrics + SET + target_project_id = merge_requests_batch.target_project_id + FROM merge_requests_batch + WHERE merge_request_metrics.merge_request_id=merge_requests_batch.id + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb deleted file mode 100644 index 103bd98af14..00000000000 --- a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class FillFileStoreJobArtifact - class JobArtifact < ActiveRecord::Base - self.table_name = 'ci_job_artifacts' - end - - def perform(start_id, stop_id) - FillFileStoreJobArtifact::JobArtifact - .where(file_store: nil) - .where(id: (start_id..stop_id)) - .update_all(file_store: 1) - end - end - end -end diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb deleted file mode 100644 index 77c1f1ffaf0..00000000000 --- a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class FillFileStoreLfsObject - class LfsObject < ActiveRecord::Base - self.table_name = 'lfs_objects' - end - - def perform(start_id, stop_id) - FillFileStoreLfsObject::LfsObject - .where(file_store: nil) - .where(id: (start_id..stop_id)) - .update_all(file_store: 1) - end - end - end -end diff --git a/lib/gitlab/background_migration/fill_store_upload.rb b/lib/gitlab/background_migration/fill_store_upload.rb deleted file mode 100644 index cba3e21cea6..00000000000 --- a/lib/gitlab/background_migration/fill_store_upload.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class FillStoreUpload - class Upload < ActiveRecord::Base - self.table_name = 'uploads' - self.inheritance_column = :_type_disabled - end - - def perform(start_id, stop_id) - FillStoreUpload::Upload - .where(store: nil) - .where(id: (start_id..stop_id)) - .update_all(store: 1) - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_cross_project_label_links.rb b/lib/gitlab/background_migration/fix_cross_project_label_links.rb deleted file mode 100644 index 20a98c8e141..00000000000 --- a/lib/gitlab/background_migration/fix_cross_project_label_links.rb +++ /dev/null @@ -1,140 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class FixCrossProjectLabelLinks - GROUP_NESTED_LEVEL = 10.freeze - - class Project < ActiveRecord::Base - self.table_name = 'projects' - end - - class Label < ActiveRecord::Base - self.inheritance_column = :_type_disabled - self.table_name = 'labels' - end - - class LabelLink < ActiveRecord::Base - self.table_name = 'label_links' - end - - class Issue < ActiveRecord::Base - self.table_name = 'issues' - end - - class MergeRequest < ActiveRecord::Base - self.table_name = 'merge_requests' - end - - class Namespace < ActiveRecord::Base - self.inheritance_column = :_type_disabled - self.table_name = 'namespaces' - - def self.groups_with_descendants_ids(start_id, stop_id) - # To isolate migration code, we avoid usage of - # Gitlab::GroupHierarchy#base_and_descendants which already - # does this job better - ids = Namespace.where(type: 'Group', id: Label.where(type: 'GroupLabel').select('distinct group_id')).where(id: start_id..stop_id).pluck(:id) - group_ids = ids - - GROUP_NESTED_LEVEL.times do - ids = Namespace.where(type: 'Group', parent_id: ids).pluck(:id) - break if ids.empty? - - group_ids += ids - end - - group_ids.uniq - end - end - - def perform(start_id, stop_id) - group_ids = Namespace.groups_with_descendants_ids(start_id, stop_id) - project_ids = Project.where(namespace_id: group_ids).select(:id) - - fix_issues(project_ids) - fix_merge_requests(project_ids) - end - - private - - # select IDs of issues which reference a label which is: - # a) a project label of a different project, or - # b) a group label of a different group than issue's project group - def fix_issues(project_ids) - issue_ids = Label - .joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'Issue\' - INNER JOIN issues ON issues.id = label_links.target_id - INNER JOIN projects ON projects.id = issues.project_id') - .where('issues.project_id in (?)', project_ids) - .where('(labels.project_id is not null and labels.project_id != issues.project_id) '\ - 'or (labels.group_id is not null and labels.group_id != projects.namespace_id)') - .select('distinct issues.id') - - Issue.where(id: issue_ids).find_each { |issue| check_resource_labels(issue, issue.project_id) } - end - - # select IDs of MRs which reference a label which is: - # a) a project label of a different project, or - # b) a group label of a different group than MR's project group - def fix_merge_requests(project_ids) - mr_ids = Label - .joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'MergeRequest\' - INNER JOIN merge_requests ON merge_requests.id = label_links.target_id - INNER JOIN projects ON projects.id = merge_requests.target_project_id') - .where('merge_requests.target_project_id in (?)', project_ids) - .where('(labels.project_id is not null and labels.project_id != merge_requests.target_project_id) '\ - 'or (labels.group_id is not null and labels.group_id != projects.namespace_id)') - .select('distinct merge_requests.id') - - MergeRequest.where(id: mr_ids).find_each { |merge_request| check_resource_labels(merge_request, merge_request.target_project_id) } - end - - def check_resource_labels(resource, project_id) - local_labels = available_labels(project_id) - - # get all label links for the given resource (issue/MR) - # which reference a label not included in available_labels - # (other than its project labels and labels of ancestor groups) - cross_labels = LabelLink - .select('label_id, labels.title as title, labels.color as color, label_links.id as label_link_id') - .joins('INNER JOIN labels ON labels.id = label_links.label_id') - .where(target_type: resource.class.name.demodulize, target_id: resource.id) - .where('labels.id not in (?)', local_labels.select(:id)) - - cross_labels.each do |label| - matching_label = local_labels.find {|l| l.title == label.title && l.color == label.color} - - next unless matching_label - - Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}" # rubocop:disable Gitlab/RailsLogger - LabelLink.update(label.label_link_id, label_id: matching_label.id) - end - end - - # get all labels available for the project (including - # group labels of ancestor groups) - def available_labels(project_id) - @labels ||= {} - @labels[project_id] ||= Label - .where("(type = 'GroupLabel' and group_id in (?)) or (type = 'ProjectLabel' and id = ?)", - project_group_ids(project_id), - project_id) - end - - def project_group_ids(project_id) - ids = [Project.find(project_id).namespace_id] - - GROUP_NESTED_LEVEL.times do - group = Namespace.find(ids.last) - break unless group.parent_id - - ids << group.parent_id - end - - ids - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb deleted file mode 100644 index 268c6083d3c..00000000000 --- a/lib/gitlab/background_migration/migrate_build_stage.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MigrateBuildStage - module Migratable - class Stage < ActiveRecord::Base - self.table_name = 'ci_stages' - end - - class Build < ActiveRecord::Base - self.table_name = 'ci_builds' - self.inheritance_column = :_type_disabled - - def ensure_stage!(attempts: 2) - find_stage || create_stage! - rescue ActiveRecord::RecordNotUnique - retry if (attempts -= 1) > 0 - raise - end - - def find_stage - Stage.find_by(name: self.stage || 'test', - pipeline_id: self.commit_id, - project_id: self.project_id) - end - - def create_stage! - Stage.create!(name: self.stage || 'test', - pipeline_id: self.commit_id, - project_id: self.project_id) - end - end - end - - def perform(start_id, stop_id) - stages = Migratable::Build.where('stage_id IS NULL') - .where('id BETWEEN ? AND ?', start_id, stop_id) - .map { |build| build.ensure_stage! } - .compact.map(&:id) - - MigrateBuildStageIdReference.new.perform(start_id, stop_id) - MigrateStageStatus.new.perform(stages.min, stages.max) - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb deleted file mode 100644 index 0a8a4313cd5..00000000000 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MigrateBuildStageIdReference - def perform(start_id, stop_id) - sql = <<-SQL.strip_heredoc - UPDATE ci_builds - SET stage_id = - (SELECT id FROM ci_stages - WHERE ci_stages.pipeline_id = ci_builds.commit_id - AND ci_stages.name = ci_builds.stage) - WHERE ci_builds.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} - AND ci_builds.stage_id IS NULL - SQL - - ActiveRecord::Base.connection.execute(sql) - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_stage_index.rb b/lib/gitlab/background_migration/migrate_stage_index.rb deleted file mode 100644 index 55608529cee..00000000000 --- a/lib/gitlab/background_migration/migrate_stage_index.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MigrateStageIndex - def perform(start_id, stop_id) - migrate_stage_index_sql(start_id.to_i, stop_id.to_i).tap do |sql| - ActiveRecord::Base.connection.execute(sql) - end - end - - private - - def migrate_stage_index_sql(start_id, stop_id) - <<~SQL - WITH freqs AS ( - SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds - WHERE stage_id BETWEEN #{start_id} AND #{stop_id} - AND stage_idx IS NOT NULL - GROUP BY stage_id, stage_idx - ), indexes AS ( - SELECT DISTINCT stage_id, first_value(stage_idx) - OVER (PARTITION BY stage_id ORDER BY freq DESC) AS index - FROM freqs - ) - - UPDATE ci_stages SET position = indexes.index - FROM indexes WHERE indexes.stage_id = ci_stages.id - AND ci_stages.position IS NULL; - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb deleted file mode 100644 index fcbcaacb2d6..00000000000 --- a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true -# -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class PopulateClusterKubernetesNamespaceTable - include Gitlab::Database::MigrationHelpers - - BATCH_SIZE = 1_000 - - module Migratable - class KubernetesNamespace < ActiveRecord::Base - self.table_name = 'clusters_kubernetes_namespaces' - end - - class ClusterProject < ActiveRecord::Base - include EachBatch - - self.table_name = 'cluster_projects' - - belongs_to :project - - def self.with_no_kubernetes_namespace - where.not(id: Migratable::KubernetesNamespace.select(:cluster_project_id)) - end - - def namespace - slug = "#{project.path}-#{project.id}".downcase - slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '') - end - - def service_account - "#{namespace}-service-account" - end - end - - class Project < ActiveRecord::Base - self.table_name = 'projects' - end - end - - def perform - cluster_projects_with_no_kubernetes_namespace.each_batch(of: BATCH_SIZE) do |cluster_projects_batch, index| - sql_values = sql_values_for(cluster_projects_batch) - - insert_into_cluster_kubernetes_namespace(sql_values) - end - end - - private - - def cluster_projects_with_no_kubernetes_namespace - Migratable::ClusterProject.with_no_kubernetes_namespace - end - - def sql_values_for(cluster_projects) - cluster_projects.map do |cluster_project| - values_for_cluster_project(cluster_project) - end - end - - def values_for_cluster_project(cluster_project) - { - cluster_project_id: cluster_project.id, - cluster_id: cluster_project.cluster_id, - project_id: cluster_project.project_id, - namespace: cluster_project.namespace, - service_account_name: cluster_project.service_account, - created_at: 'NOW()', - updated_at: 'NOW()' - } - end - - def insert_into_cluster_kubernetes_namespace(rows) - Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name, # rubocop:disable Gitlab/BulkInsert - rows, - disable_quote: [:created_at, :updated_at]) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb b/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb new file mode 100644 index 00000000000..e8f436b183e --- /dev/null +++ b/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class creates/updates those personal snippets statistics + # that haven't been created nor initialized. + # It also updates the related root storage namespace stats + class PopulatePersonalSnippetStatistics + def perform(snippet_ids) + personal_snippets(snippet_ids).group_by(&:author).each do |author, author_snippets| + upsert_snippet_statistics(author_snippets) + update_namespace_statistics(author.namespace) + end + end + + private + + def personal_snippets(snippet_ids) + PersonalSnippet + .where(id: snippet_ids) + .includes(author: :namespace) + .includes(:statistics) + .includes(snippet_repository: :shard) + end + + def upsert_snippet_statistics(snippets) + snippets.each do |snippet| + response = Snippets::UpdateStatisticsService.new(snippet).execute + + error_message("#{response.message} snippet: #{snippet.id}") if response.error? + end + end + + def update_namespace_statistics(namespace) + Namespaces::StatisticsRefresherService.new.execute(namespace) + rescue => e + error_message("Error updating statistics for namespace #{namespace.id}: #{e.message}") + end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + + def error_message(message) + logger.error(message: "Snippet Statistics Migration: #{message}") + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb deleted file mode 100644 index 43698b7955f..00000000000 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ /dev/null @@ -1,111 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class processes a batch of rows in `untracked_files_for_uploads` by - # adding each file to the `uploads` table if it does not exist. - class PopulateUntrackedUploads - def perform(start_id, end_id) - return unless migrate? - - files = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.where(id: start_id..end_id) - processed_files = insert_uploads_if_needed(files) - processed_files.delete_all - - drop_temp_table_if_finished - end - - private - - def migrate? - Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.table_exists? && - Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Upload.table_exists? - end - - def insert_uploads_if_needed(files) - filtered_files, error_files = filter_error_files(files) - filtered_files = filter_existing_uploads(filtered_files) - filtered_files = filter_deleted_models(filtered_files) - insert(filtered_files) - - processed_files = files.where.not(id: error_files.map(&:id)) - processed_files - end - - def filter_error_files(files) - files.partition do |file| - file.to_h - true - rescue => e - msg = <<~MSG - Error parsing path "#{file.path}": - #{e.message} - #{e.backtrace.join("\n ")} - MSG - Rails.logger.error(msg) # rubocop:disable Gitlab/RailsLogger - false - end - end - - def filter_existing_uploads(files) - paths = files.map(&:upload_path) - existing_paths = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Upload.where(path: paths).pluck(:path).to_set - - files.reject do |file| - existing_paths.include?(file.upload_path) - end - end - - # There are files on disk that are not in the uploads table because their - # model was deleted, and we don't delete the files on disk. - def filter_deleted_models(files) - ids = deleted_model_ids(files) - - files.reject do |file| - ids[file.model_type].include?(file.model_id) - end - end - - def deleted_model_ids(files) - ids = { - 'Appearance' => [], - 'Namespace' => [], - 'Note' => [], - 'Project' => [], - 'User' => [] - } - - # group model IDs by model type - files.each do |file| - ids[file.model_type] << file.model_id - end - - ids.each do |model_type, model_ids| - model_class = "Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::#{model_type}".constantize - found_ids = model_class.where(id: model_ids.uniq).pluck(:id) - deleted_ids = ids[model_type] - found_ids - ids[model_type] = deleted_ids - end - - ids - end - - def insert(files) - rows = files.map do |file| - file.to_h.merge(created_at: 'NOW()') - end - - Gitlab::Database.bulk_insert('uploads', # rubocop:disable Gitlab/BulkInsert - rows, - disable_quote: :created_at) - end - - def drop_temp_table_if_finished - if Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.all.empty? && !Rails.env.test? # Dropping a table intermittently breaks test cleanup - Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.connection.drop_table(:untracked_files_for_uploads, - if_exists: true) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb deleted file mode 100644 index 23e8be4a9ab..00000000000 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ /dev/null @@ -1,190 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module BackgroundMigration - module PopulateUntrackedUploadsDependencies - # This class is responsible for producing the attributes necessary to - # track an uploaded file in the `uploads` table. - class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength - self.table_name = 'untracked_files_for_uploads' - - # Ends with /:random_hex/:filename - FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z}.freeze - FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/.freeze - - # These regex patterns are tested against a relative path, relative to - # the upload directory. - # For convenience, if there exists a capture group in the pattern, then - # it indicates the model_id. - PATH_PATTERNS = [ - { - pattern: %r{\A-/system/appearance/logo/(\d+)/}, - uploader: 'AttachmentUploader', - model_type: 'Appearance' - }, - { - pattern: %r{\A-/system/appearance/header_logo/(\d+)/}, - uploader: 'AttachmentUploader', - model_type: 'Appearance' - }, - { - pattern: %r{\A-/system/note/attachment/(\d+)/}, - uploader: 'AttachmentUploader', - model_type: 'Note' - }, - { - pattern: %r{\A-/system/user/avatar/(\d+)/}, - uploader: 'AvatarUploader', - model_type: 'User' - }, - { - pattern: %r{\A-/system/group/avatar/(\d+)/}, - uploader: 'AvatarUploader', - model_type: 'Namespace' - }, - { - pattern: %r{\A-/system/project/avatar/(\d+)/}, - uploader: 'AvatarUploader', - model_type: 'Project' - }, - { - pattern: FILE_UPLOADER_PATH, - uploader: 'FileUploader', - model_type: 'Project' - } - ].freeze - - def to_h - @upload_hash ||= { - path: upload_path, - uploader: uploader, - model_type: model_type, - model_id: model_id, - size: file_size, - checksum: checksum - } - end - - def upload_path - # UntrackedFile#path is absolute, but Upload#path depends on uploader - @upload_path ||= - if uploader == 'FileUploader' - # Path relative to project directory in uploads - matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH) - matchd[0].sub(%r{\A/}, '') # remove leading slash - else - path - end - end - - def uploader - matching_pattern_map[:uploader] - end - - def model_type - matching_pattern_map[:model_type] - end - - def model_id - return @model_id if defined?(@model_id) - - pattern = matching_pattern_map[:pattern] - matchd = path_relative_to_upload_dir.match(pattern) - - # If something is captured (matchd[1] is not nil), it is a model_id - # Only the FileUploader pattern will not match an ID - @model_id = matchd[1] ? matchd[1].to_i : file_uploader_model_id - end - - def file_size - File.size(absolute_path) - end - - def checksum - Digest::SHA256.file(absolute_path).hexdigest - end - - private - - def matching_pattern_map - @matching_pattern_map ||= PATH_PATTERNS.find do |path_pattern_map| - path_relative_to_upload_dir.match(path_pattern_map[:pattern]) - end - - unless @matching_pattern_map - raise "Unknown upload path pattern \"#{path}\"" - end - - @matching_pattern_map - end - - def file_uploader_model_id - matchd = path_relative_to_upload_dir.match(FULL_PATH_CAPTURE) - not_found_msg = <<~MSG - Could not capture project full_path from a FileUploader path: - "#{path_relative_to_upload_dir}" - MSG - raise not_found_msg unless matchd - - full_path = matchd[1] - project = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Project.find_by_full_path(full_path) - return unless project - - project.id - end - - # Not including a leading slash - def path_relative_to_upload_dir - upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR - base = %r{\A#{Regexp.escape(upload_dir)}/} - @path_relative_to_upload_dir ||= path.sub(base, '') - end - - def absolute_path - File.join(Gitlab.config.uploads.storage_path, path) - end - end - - # Avoid using application code - class Upload < ActiveRecord::Base - self.table_name = 'uploads' - end - - # Avoid using application code - class Appearance < ActiveRecord::Base - self.table_name = 'appearances' - end - - # Avoid using application code - class Namespace < ActiveRecord::Base - self.table_name = 'namespaces' - end - - # Avoid using application code - class Note < ActiveRecord::Base - self.table_name = 'notes' - end - - # Avoid using application code - class User < ActiveRecord::Base - self.table_name = 'users' - end - - # Since project Markdown upload paths don't contain the project ID, we have to find the - # project by its full_path. Due to MySQL/PostgreSQL differences, and historical reasons, - # the logic is somewhat complex, so I've mostly copied it in here. - class Project < ActiveRecord::Base - self.table_name = 'projects' - - def self.find_by_full_path(path) - order_sql = Arel.sql("(CASE WHEN routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)") - where_full_path_in(path).reorder(order_sql).take - end - - def self.where_full_path_in(path) - where = "(LOWER(routes.path) = LOWER(#{connection.quote(path)}))" - joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'").where(where) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb deleted file mode 100644 index 3d943205783..00000000000 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ /dev/null @@ -1,173 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class finds all non-hashed uploaded file paths and saves them to a - # `untracked_files_for_uploads` table. - class PrepareUntrackedUploads # rubocop:disable Metrics/ClassLength - # For bulk_queue_background_migration_jobs_by_range - include Database::MigrationHelpers - include ::Gitlab::Utils::StrongMemoize - - FIND_BATCH_SIZE = 500 - RELATIVE_UPLOAD_DIR = "uploads" - ABSOLUTE_UPLOAD_DIR = File.join( - Gitlab.config.uploads.storage_path, - RELATIVE_UPLOAD_DIR - ) - FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads' - START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/}.freeze - EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*" - EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*" - - # This class is used to iterate over batches of - # `untracked_files_for_uploads` rows. - class UntrackedFile < ActiveRecord::Base - include EachBatch - - self.table_name = 'untracked_files_for_uploads' - end - - def perform - ensure_temporary_tracking_table_exists - - # Since Postgres < 9.5 does not have ON CONFLICT DO NOTHING, and since - # doing inserts-if-not-exists without ON CONFLICT DO NOTHING would be - # slow, start with an empty table for Postgres < 9.5. - # That way we can do bulk inserts at ~30x the speed of individual - # inserts (~20 minutes worth of inserts at GitLab.com scale instead of - # ~10 hours). - # In all other cases, installations will get both bulk inserts and the - # ability for these jobs to retry without having to clear and reinsert. - clear_untracked_file_paths unless can_bulk_insert_and_ignore_duplicates? - - store_untracked_file_paths - - if UntrackedFile.all.empty? - drop_temp_table - else - schedule_populate_untracked_uploads_jobs - end - end - - private - - def ensure_temporary_tracking_table_exists - table_name = :untracked_files_for_uploads - - unless ActiveRecord::Base.connection.table_exists?(table_name) - UntrackedFile.connection.create_table table_name do |t| - t.string :path, limit: 600, null: false - t.index :path, unique: true - end - end - end - - def clear_untracked_file_paths - UntrackedFile.delete_all - end - - def store_untracked_file_paths - return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR) - - each_file_batch(ABSOLUTE_UPLOAD_DIR, FIND_BATCH_SIZE) do |file_paths| - insert_file_paths(file_paths) - end - end - - def each_file_batch(search_dir, batch_size, &block) - cmd = build_find_command(search_dir) - - Open3.popen2(*cmd) do |stdin, stdout, status_thread| - yield_paths_in_batches(stdout, batch_size, &block) - - raise "Find command failed" unless status_thread.value.success? - end - end - - def yield_paths_in_batches(stdout, batch_size, &block) - paths = [] - - stdout.each_line("\0") do |line| - paths << line.chomp("\0").sub(START_WITH_ROOT_REGEX, '') - - if paths.size >= batch_size - yield(paths) - paths = [] - end - end - - yield(paths) if paths.any? - end - - def build_find_command(search_dir) - cmd = %W[find -L #{search_dir} - -type f - ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) - ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) - -print0] - - ionice = which_ionice - cmd = %W[#{ionice} -c Idle] + cmd if ionice - - log_msg = "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" - Rails.logger.info log_msg # rubocop:disable Gitlab/RailsLogger - - cmd - end - - def which_ionice - Gitlab::Utils.which('ionice') - rescue StandardError - # In this case, returning false is relatively safe, - # even though it isn't very nice - false - end - - def insert_file_paths(file_paths) - sql = insert_sql(file_paths) - - ActiveRecord::Base.connection.execute(sql) - end - - def insert_sql(file_paths) - if postgresql_pre_9_5? - "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" - else - "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\ - " ON CONFLICT DO NOTHING;" - end - end - - def table_columns_and_values_for_insert(file_paths) - values = file_paths.map do |file_path| - ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend - end.join(', ') - - "#{UntrackedFile.table_name} (path) VALUES #{values}" - end - - def can_bulk_insert_and_ignore_duplicates? - !postgresql_pre_9_5? - end - - def postgresql_pre_9_5? - strong_memoize(:postgresql_pre_9_5) do - Gitlab::Database.version.to_f < 9.5 - end - end - - def schedule_populate_untracked_uploads_jobs - bulk_queue_background_migration_jobs_by_range( - UntrackedFile, FOLLOW_UP_MIGRATION) - end - - def drop_temp_table - unless Rails.env.test? # Dropping a table intermittently breaks test cleanup - UntrackedFile.connection.drop_table(:untracked_files_for_uploads, - if_exists: true) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb deleted file mode 100644 index 9ef6d8654ae..00000000000 --- a/lib/gitlab/background_migration/remove_restricted_todos.rb +++ /dev/null @@ -1,158 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation -# rubocop:disable Metrics/ClassLength - -module Gitlab - module BackgroundMigration - class RemoveRestrictedTodos - PRIVATE_FEATURE = 10 - PRIVATE_PROJECT = 0 - - class Project < ActiveRecord::Base - self.table_name = 'projects' - end - - class ProjectAuthorization < ActiveRecord::Base - self.table_name = 'project_authorizations' - end - - class ProjectFeature < ActiveRecord::Base - self.table_name = 'project_features' - end - - class Todo < ActiveRecord::Base - include EachBatch - - self.table_name = 'todos' - end - - class Issue < ActiveRecord::Base - include EachBatch - - self.table_name = 'issues' - end - - def perform(start_id, stop_id) - projects = Project.where('EXISTS (SELECT 1 FROM todos WHERE todos.project_id = projects.id)') - .where(id: start_id..stop_id) - - projects.each do |project| - remove_confidential_issue_todos(project.id) - - if project.visibility_level == PRIVATE_PROJECT - remove_non_members_todos(project.id) - else - remove_restricted_features_todos(project.id) - end - end - end - - private - - def remove_non_members_todos(project_id) - batch_remove_todos_cte(project_id) - end - - def remove_confidential_issue_todos(project_id) - # min access level to access a confidential issue is reporter - min_reporters = authorized_users(project_id) - .select(:user_id) - .where('access_level >= ?', 20) - - confidential_issues = Issue.select(:id, :author_id).where(confidential: true, project_id: project_id) - confidential_issues.each_batch(of: 100, order_hint: :confidential) do |batch| - batch.each do |issue| - assigned_users = IssueAssignee.select(:user_id).where(issue_id: issue.id) - - todos = Todo.where(target_type: 'Issue', target_id: issue.id) - .where('user_id NOT IN (?)', min_reporters) - .where('user_id NOT IN (?)', assigned_users) - todos = todos.where('user_id != ?', issue.author_id) if issue.author_id - - todos.delete_all - end - end - end - - def remove_restricted_features_todos(project_id) - ProjectFeature.where(project_id: project_id).each do |project_features| - target_types = [] - target_types << 'Issue' if private?(project_features.issues_access_level) - target_types << 'MergeRequest' if private?(project_features.merge_requests_access_level) - target_types << 'Commit' if private?(project_features.repository_access_level) - - next if target_types.empty? - - batch_remove_todos_cte(project_id, target_types) - end - end - - def private?(feature_level) - feature_level == PRIVATE_FEATURE - end - - def authorized_users(project_id) - ProjectAuthorization.select(:user_id).where(project_id: project_id) - end - - def unauthorized_project_todos(project_id) - Todo.where(project_id: project_id) - .where('user_id NOT IN (?)', authorized_users(project_id)) - end - - def batch_remove_todos_cte(project_id, target_types = nil) - loop do - count = remove_todos_cte(project_id, target_types) - - break if count == 0 - end - end - - def remove_todos_cte(project_id, target_types = nil) - sql = [] - sql << with_all_todos_sql(project_id, target_types) - sql << as_deleted_sql - sql << "SELECT count(*) FROM deleted" - - result = Todo.connection.exec_query(sql.join(' ')) - result.rows[0][0].to_i - end - - def with_all_todos_sql(project_id, target_types = nil) - if target_types - table = Arel::Table.new(:todos) - in_target = table[:target_type].in(target_types) - target_types_sql = " AND #{in_target.to_sql}" - end - - <<-SQL - WITH all_todos AS ( - SELECT id - FROM "todos" - WHERE "todos"."project_id" = #{project_id} - AND (user_id NOT IN ( - SELECT "project_authorizations"."user_id" - FROM "project_authorizations" - WHERE "project_authorizations"."project_id" = #{project_id}) - #{target_types_sql} - ) - ), - SQL - end - - def as_deleted_sql - <<-SQL - deleted AS ( - DELETE FROM todos - WHERE id IN ( - SELECT id - FROM all_todos - LIMIT 5000 - ) - RETURNING id - ) - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb deleted file mode 100644 index bc434b0cb64..00000000000 --- a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - # Ensures services which previously received all notes events continue - # to receive confidential ones. - class SetConfidentialNoteEventsOnServices - class Service < ActiveRecord::Base - self.table_name = 'services' - - include ::EachBatch - - def self.services_to_update - where(confidential_note_events: nil, note_events: true) - end - end - - def perform(start_id, stop_id) - Service.services_to_update - .where(id: start_id..stop_id) - .update_all(confidential_note_events: true) - end - end - end -end diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb deleted file mode 100644 index 28d8d2c640b..00000000000 --- a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - # Ensures hooks which previously received all notes events continue - # to receive confidential ones. - class SetConfidentialNoteEventsOnWebhooks - class WebHook < ActiveRecord::Base - self.table_name = 'web_hooks' - - include ::EachBatch - - def self.hooks_to_update - where(confidential_note_events: nil, note_events: true) - end - end - - def perform(start_id, stop_id) - WebHook.hooks_to_update - .where(id: start_id..stop_id) - .update_all(confidential_note_events: true) - end - end - end -end diff --git a/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb b/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb new file mode 100644 index 00000000000..9f765d03d62 --- /dev/null +++ b/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Sets the MergeRequestDiff#files_count value for old rows + class SetMergeRequestDiffFilesCount + COUNT_SUBQUERY = <<~SQL + files_count = ( + SELECT count(*) + FROM merge_request_diff_files + WHERE merge_request_diff_files.merge_request_diff_id = merge_request_diffs.id + ) + SQL + + class MergeRequestDiff < ActiveRecord::Base # rubocop:disable Style/Documentation + include EachBatch + + self.table_name = 'merge_request_diffs' + end + + def perform(start_id, end_id) + MergeRequestDiff.where(id: start_id..end_id).each_batch do |relation| + relation.update_all(COUNT_SUBQUERY) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/set_null_external_diff_store_to_local_value.rb b/lib/gitlab/background_migration/set_null_external_diff_store_to_local_value.rb new file mode 100644 index 00000000000..71f3483987e --- /dev/null +++ b/lib/gitlab/background_migration/set_null_external_diff_store_to_local_value.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class is responsible for migrating a range of merge request diffs + # with external_diff_store == NULL to 1. + # + # The index `index_merge_request_diffs_external_diff_store_is_null` is + # expected to be used to find the rows here and in the migration scheduling + # the jobs that run this class. + class SetNullExternalDiffStoreToLocalValue + LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL + + # Temporary AR class for merge request diffs + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + end + + def perform(start_id, stop_id) + MergeRequestDiff.where(external_diff_store: nil, id: start_id..stop_id).update_all(external_diff_store: LOCAL_STORE) + end + end + end +end diff --git a/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb b/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb new file mode 100644 index 00000000000..9ac92aab637 --- /dev/null +++ b/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class is responsible for migrating a range of package files + # with file_store == NULL to 1. + # + # The index `index_packages_package_files_file_store_is_null` is + # expected to be used to find the rows here and in the migration scheduling + # the jobs that run this class. + class SetNullPackageFilesFileStoreToLocalValue + LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL + + # Temporary AR class for package files + class PackageFile < ActiveRecord::Base + self.table_name = 'packages_package_files' + end + + def perform(start_id, stop_id) + Packages::PackageFile.where(file_store: nil, id: start_id..stop_id).update_all(file_store: LOCAL_STORE) + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb index d71a50a0af6..b3876018553 100644 --- a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb @@ -12,26 +12,22 @@ module Gitlab ISOLATION_MODULE = 'Gitlab::BackgroundMigration::UserMentions::Models' def perform(resource_model, join, conditions, with_notes, start_id, end_id) + return unless Feature.enabled?(:migrate_user_mentions, default_enabled: true) + resource_model = "#{ISOLATION_MODULE}::#{resource_model}".constantize if resource_model.is_a?(String) model = with_notes ? Gitlab::BackgroundMigration::UserMentions::Models::Note : resource_model resource_user_mention_model = resource_model.user_mention_model records = model.joins(join).where(conditions).where(id: start_id..end_id) - records.in_groups_of(BULK_INSERT_SIZE, false).each do |records| + records.each_batch(of: BULK_INSERT_SIZE) do |records| mentions = [] records.each do |record| mention_record = record.build_mention_values(resource_user_mention_model.resource_foreign_key) mentions << mention_record unless mention_record.blank? end - Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert - resource_user_mention_model.table_name, - mentions, - return_ids: true, - disable_quote: resource_model.no_quote_columns, - on_conflict: :do_nothing - ) + resource_user_mention_model.insert_all(mentions) unless mentions.empty? end end end diff --git a/lib/gitlab/background_migration/user_mentions/models/commit.rb b/lib/gitlab/background_migration/user_mentions/models/commit.rb index 279e93dbf0d..65f4a7a25b6 100644 --- a/lib/gitlab/background_migration/user_mentions/models/commit.rb +++ b/lib/gitlab/background_migration/user_mentions/models/commit.rb @@ -6,6 +6,7 @@ module Gitlab module UserMentions module Models class Commit + include EachBatch include Concerns::IsolatedMentionable include Concerns::MentionableMigrationMethods diff --git a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb index 0cdfc6447c7..bdb90b5d2b9 100644 --- a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb +++ b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb @@ -7,6 +7,7 @@ module Gitlab module Models module DesignManagement class Design < ActiveRecord::Base + include EachBatch include Concerns::MentionableMigrationMethods def self.user_mention_model diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb index dc2b7819800..61d9244a4c9 100644 --- a/lib/gitlab/background_migration/user_mentions/models/epic.rb +++ b/lib/gitlab/background_migration/user_mentions/models/epic.rb @@ -6,6 +6,7 @@ module Gitlab module UserMentions module Models class Epic < ActiveRecord::Base + include EachBatch include Concerns::IsolatedMentionable include Concerns::MentionableMigrationMethods include CacheMarkdownField diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb index 655c1db71ae..6b52afea17c 100644 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb +++ b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb @@ -6,6 +6,7 @@ module Gitlab module UserMentions module Models class MergeRequest < ActiveRecord::Base + include EachBatch include Concerns::IsolatedMentionable include CacheMarkdownField include Concerns::MentionableMigrationMethods diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb index c32292ad704..a3224c8c456 100644 --- a/lib/gitlab/background_migration/user_mentions/models/note.rb +++ b/lib/gitlab/background_migration/user_mentions/models/note.rb @@ -6,6 +6,7 @@ module Gitlab module UserMentions module Models class Note < ActiveRecord::Base + include EachBatch include Concerns::IsolatedMentionable include CacheMarkdownField diff --git a/lib/gitlab/backtrace_cleaner.rb b/lib/gitlab/backtrace_cleaner.rb index 30ec99808f7..d04f0983d12 100644 --- a/lib/gitlab/backtrace_cleaner.rb +++ b/lib/gitlab/backtrace_cleaner.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# Remove some GitLab code from backtraces. Do not use this for logging errors in +# production environments, as the error may be thrown by our middleware. module Gitlab module BacktraceCleaner IGNORE_BACKTRACES = %w[ diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 144ba2ec031..ab7a08ffef9 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -123,7 +123,7 @@ module Gitlab cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all) output, status = Gitlab::Popen.popen(cmd) - raise output unless status.zero? + raise output unless status == 0 bundle_path end diff --git a/lib/gitlab/build_access.rb b/lib/gitlab/build_access.rb index 37e79413541..81759693749 100644 --- a/lib/gitlab/build_access.rb +++ b/lib/gitlab/build_access.rb @@ -2,8 +2,6 @@ module Gitlab class BuildAccess < UserAccess - attr_accessor :user, :project - # This bypasses the `can?(:access_git)`-check we normally do in `UserAccess` # for CI. That way if a user was able to trigger a pipeline, then the # build is allowed to clone the project. diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 8bb5ac94e45..67c777f67a7 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -25,7 +25,7 @@ module Gitlab @logger.append_message("Running checks for ref: #{@branch_name || @tag_name}") end - def exec + def validate! ref_level_checks # Check of commits should happen as the last step # given they're expensive in terms of performance diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index f81c215d847..b70a6a69b93 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -7,7 +7,10 @@ module Gitlab ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' def validate! + # This feature flag is used for disabling integrify check on some envs + # because these costy calculations may cause performance issues return unless Feature.enabled?(:lfs_check, default_enabled: true) + return unless project.lfs_enabled? return if skip_lfs_integrity_check diff --git a/lib/gitlab/ci/build/artifacts/expire_in_parser.rb b/lib/gitlab/ci/build/artifacts/expire_in_parser.rb new file mode 100644 index 00000000000..3e8a1fb86fc --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/expire_in_parser.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Artifacts + class ExpireInParser + def self.validate_duration(value) + new(value).validate_duration + end + + def initialize(value) + @value = value + end + + def validate_duration + return true if never? + + parse + rescue ChronicDuration::DurationParseError + false + end + + def seconds_from_now + parse&.seconds&.from_now + end + + private + + attr_reader :value + + def parse + return if never? + + ChronicDuration.parse(value) + end + + def never? + value.to_s.casecmp('never') == 0 + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index ef354832e8e..355fffbf9c6 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -16,7 +16,7 @@ module Gitlab # class Entry attr_reader :entries - attr_accessor :name + attr_writer :name def initialize(path, entries) @entries = entries diff --git a/lib/gitlab/ci/build/auto_retry.rb b/lib/gitlab/ci/build/auto_retry.rb new file mode 100644 index 00000000000..e6ef12975c2 --- /dev/null +++ b/lib/gitlab/ci/build/auto_retry.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +class Gitlab::Ci::Build::AutoRetry + include Gitlab::Utils::StrongMemoize + + DEFAULT_RETRIES = { + scheduler_failure: 2 + }.freeze + + def initialize(build) + @build = build + end + + def allowed? + return false unless @build.retryable? + + within_max_retry_limit? + end + + private + + def within_max_retry_limit? + max_allowed_retries > 0 && max_allowed_retries > @build.retries_count + end + + def max_allowed_retries + strong_memoize(:max_allowed_retries) do + options_retry_max || DEFAULT_RETRIES.fetch(@build.failure_reason.to_sym, 0) + end + end + + def options_retry_max + Integer(options_retry[:max], exception: false) if retry_on_reason_or_always? + end + + def options_retry_when + options_retry.fetch(:when, ['always']) + end + + def retry_on_reason_or_always? + options_retry_when.include?(@build.failure_reason.to_s) || + options_retry_when.include?('always') + end + + # The format of the retry option changed in GitLab 11.5: Before it was + # integer only, after it is a hash. New builds are created with the new + # format, but builds created before GitLab 11.5 and saved in database still + # have the old integer only format. This method returns the retry option + # normalized as a hash in 11.5+ format. + def options_retry + strong_memoize(:options_retry) do + value = @build.options&.dig(:retry) + value = value.is_a?(Integer) ? { max: value } : value.to_h + value.with_indifferent_access + end + end +end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index f8550b50905..3f0ccefa9e5 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -21,8 +21,6 @@ module Gitlab end def from_release(job) - return unless Gitlab::Ci::Features.release_generation_enabled? - release = job.options[:release] return unless release diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index a9a9636637f..206dbaea272 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -42,7 +42,7 @@ module Gitlab inclusion: { in: %w[on_success on_failure always], message: 'should be on_success, on_failure ' \ 'or always' } - validates :expire_in, duration: true + validates :expire_in, duration: { parser: ::Gitlab::Ci::Build::Artifacts::ExpireInParser } end end diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index f4362d3b0ce..a8b67a1db4f 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -11,7 +11,7 @@ module Gitlab class Bridge < ::Gitlab::Config::Entry::Node include ::Gitlab::Ci::Config::Entry::Processable - ALLOWED_KEYS = %i[trigger allow_failure when needs].freeze + ALLOWED_KEYS = %i[trigger].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index a615cab1a80..f960cec1f26 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,9 +11,8 @@ module Gitlab include ::Gitlab::Ci::Config::Entry::Processable ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze - ALLOWED_KEYS = %i[tags script type image services - allow_failure type when start_in artifacts cache - dependencies before_script needs after_script + ALLOWED_KEYS = %i[tags script type image services start_in artifacts + cache dependencies before_script after_script environment coverage retry parallel interruptible timeout resource_group release secrets].freeze @@ -23,18 +22,9 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs? validates :script, presence: true - validates :config, - disallowed_keys: { - in: %i[release], - message: 'release features are not enabled' - }, - unless: -> { Gitlab::Ci::Features.release_generation_enabled? } with_options allow_nil: true do validates :allow_failure, boolean: true - validates :parallel, numericality: { only_integer: true, - greater_than_or_equal_to: 2, - less_than_or_equal_to: 50 } validates :when, inclusion: { in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" @@ -124,13 +114,47 @@ module Gitlab description: 'This job will produce a release.', inherit: false + entry :parallel, Entry::Product::Parallel, + description: 'Parallel configuration for this job.', + inherit: false + attributes :script, :tags, :allow_failure, :when, :dependencies, :needs, :retry, :parallel, :start_in, :interruptible, :timeout, :resource_group, :release + Matcher = Struct.new(:name, :config) do + def applies? + job_is_not_hidden? && + config_is_a_hash? && + has_job_keys? + end + + private + + def job_is_not_hidden? + !name.to_s.start_with?('.') + end + + def config_is_a_hash? + config.is_a?(Hash) + end + + def has_job_keys? + if name == :default + config.key?(:script) + else + (ALLOWED_KEYS & config.keys).any? + end + end + end + def self.matching?(name, config) - !name.to_s.start_with?('.') && - config.is_a?(Hash) && config.key?(:script) + if Gitlab::Ci::Features.job_entry_matches_all_keys? + Matcher.new(name, config).applies? + else + !name.to_s.start_with?('.') && + config.is_a?(Hash) && config.key?(:script) + end end def self.visible? @@ -174,7 +198,7 @@ module Gitlab environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, retry: retry_defined? ? retry_value : nil, - parallel: has_parallel? ? parallel.to_i : nil, + parallel: has_parallel? ? parallel_value : nil, interruptible: interruptible_defined? ? interruptible_value : nil, timeout: has_timeout? ? ChronicDuration.parse(timeout.to_s) : nil, artifacts: artifacts_value, diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index b4539475d88..f10c509d0cc 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -14,7 +14,8 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Inheritable - PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables inherit].freeze + PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables + inherit allow_failure when needs].freeze included do validations do @@ -82,8 +83,8 @@ module Gitlab @entries.delete(:except) unless except_defined? # rubocop:disable Gitlab/ModuleWithInstanceVariables end - if has_rules? && !has_workflow_rules && Gitlab::Ci::Features.raise_job_rules_without_workflow_rules_warning? - add_warning('uses `rules` without defining `workflow:rules`') + unless has_workflow_rules + validate_against_warnings end # inherit root variables @@ -93,6 +94,19 @@ module Gitlab end end + def validate_against_warnings + # If rules are valid format and workflow rules are not specified + return unless rules_value + return unless Gitlab::Ci::Features.raise_job_rules_without_workflow_rules_warning? + + last_rule = rules_value.last + + if last_rule&.keys == [:when] && last_rule[:when] != 'never' + docs_url = 'read more: https://docs.gitlab.com/ee/ci/troubleshooting.html#pipeline-warnings' + add_warning("may allow multiple pipelines to run for a single action due to `rules:when` clause with no `workflow:rules` - #{docs_url}") + end + end + def name metadata[:name] end diff --git a/lib/gitlab/ci/config/entry/product/matrix.rb b/lib/gitlab/ci/config/entry/product/matrix.rb new file mode 100644 index 00000000000..6af809d46c1 --- /dev/null +++ b/lib/gitlab/ci/config/entry/product/matrix.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents matrix style parallel builds. + # + module Product + class Matrix < ::Gitlab::Config::Entry::Node + include ::Gitlab::Utils::StrongMemoize + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + validations do + validates :config, array_of_hashes: true + + validate on: :composed do + limit = Entry::Product::Parallel::PARALLEL_LIMIT + + if number_of_generated_jobs > limit + errors.add(:config, "generates too many jobs (maximum is #{limit})") + end + end + end + + def compose!(deps = nil) + super(deps) do + @config.each_with_index do |variables, index| + @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Product::Variables) + .value(variables) + .with(parent: self, description: 'matrix variables definition.') # rubocop:disable CodeReuse/ActiveRecord + .create! + end + + @entries.each_value do |entry| + entry.compose!(deps) + end + end + end + + def value + strong_memoize(:value) do + @entries.values.map(&:value) + end + end + + # rubocop:disable CodeReuse/ActiveRecord + def number_of_generated_jobs + value.sum do |config| + config.values.reduce(1) { |acc, values| acc * values.size } + end + end + # rubocop:enable CodeReuse/ActiveRecord + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/product/parallel.rb b/lib/gitlab/ci/config/entry/product/parallel.rb new file mode 100644 index 00000000000..cd9eabbbc66 --- /dev/null +++ b/lib/gitlab/ci/config/entry/product/parallel.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a parallel job config. + # + module Product + class Parallel < ::Gitlab::Config::Entry::Simplifiable + strategy :ParallelBuilds, if: -> (config) { config.is_a?(Numeric) } + strategy :MatrixBuilds, if: -> (config) { config.is_a?(Hash) } + + PARALLEL_LIMIT = 50 + + class ParallelBuilds < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, numericality: { only_integer: true, + greater_than_or_equal_to: 2, + less_than_or_equal_to: Entry::Product::Parallel::PARALLEL_LIMIT }, + allow_nil: true + end + + def value + { number: super.to_i } + end + end + + class MatrixBuilds < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Configurable + + PERMITTED_KEYS = %i[matrix].freeze + + validations do + validates :config, allowed_keys: PERMITTED_KEYS + validates :config, required_keys: PERMITTED_KEYS + end + + entry :matrix, Entry::Product::Matrix, + description: 'Variables definition for matrix builds' + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} should be an integer or a hash"] + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb new file mode 100644 index 00000000000..ac4f70fb69e --- /dev/null +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents variables for parallel matrix builds. + # + module Product + class Variables < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, variables: { array_values: true } + validates :config, length: { + minimum: 2, + too_short: 'requires at least %{count} items' + } + end + + def self.default(**) + {} + end + + def value + @config + .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } + .to_h + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 814dcc66362..cf6c2961ee7 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -54,7 +54,7 @@ module Gitlab end def execution_expired? - return false if execution_deadline.zero? + return false if execution_deadline == 0 current_monotonic_time > execution_deadline end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 1139efee9e8..451ba14bb89 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -32,7 +32,7 @@ module Gitlab return unless job_names job_names.flat_map do |job_name| - parallelized_jobs[job_name.to_sym] || job_name + parallelized_jobs[job_name.to_sym]&.map(&:name) || job_name end end @@ -42,10 +42,8 @@ module Gitlab job_needs.flat_map do |job_need| job_need_name = job_need[:name].to_sym - if all_job_names = parallelized_jobs[job_need_name] - all_job_names.map do |job_name| - job_need.merge(name: job_name) - end + if all_jobs = parallelized_jobs[job_need_name] + all_jobs.map { |job| job_need.merge(name: job.name) } else job_need end @@ -57,7 +55,7 @@ module Gitlab @jobs_config.each_with_object({}) do |(job_name, config), hash| next unless config[:parallel] - hash[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) + hash[job_name] = parallelize_job_config(job_name, config[:parallel]) end end end @@ -65,9 +63,9 @@ module Gitlab def expand_parallelize_jobs @jobs_config.each_with_object({}) do |(job_name, config), hash| if parallelized_jobs.key?(job_name) - parallelized_jobs[job_name].each_with_index do |name, index| - hash[name.to_sym] = - yield(name, config.merge(name: name, instance: index + 1)) + parallelized_jobs[job_name].each do |job| + hash[job.name.to_sym] = + yield(job.name, config.deep_merge(job.attributes)) end else hash[job_name] = yield(job_name, config) @@ -75,8 +73,8 @@ module Gitlab end end - def self.parallelize_job_names(name, total) - Array.new(total) { |index| "#{name} #{index + 1}/#{total}" } + def parallelize_job_config(name, config) + Normalizer::Factory.new(name, config).create end end end diff --git a/lib/gitlab/ci/config/normalizer/factory.rb b/lib/gitlab/ci/config/normalizer/factory.rb new file mode 100644 index 00000000000..bf813f8e878 --- /dev/null +++ b/lib/gitlab/ci/config/normalizer/factory.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + class Factory + include Gitlab::Utils::StrongMemoize + + def initialize(name, config) + @name = name + @config = config + end + + def create + return [] unless strategy + + strategy.build_from(@name, @config) + end + + private + + def strategy + strong_memoize(:strategy) do + strategies.find do |strategy| + strategy.applies_to?(@config) + end + end + end + + def strategies + [NumberStrategy, MatrixStrategy] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb new file mode 100644 index 00000000000..db21274a9ed --- /dev/null +++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + class MatrixStrategy + class << self + def applies_to?(config) + config.is_a?(Hash) && config.key?(:matrix) + end + + def build_from(job_name, initial_config) + config = expand(initial_config[:matrix]) + total = config.size + + config.map.with_index do |vars, index| + new(job_name, index.next, vars, total) + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def expand(config) + config.flat_map do |config| + values = config.values + + values[0] + .product(*values.from(1)) + .map { |vals| config.keys.zip(vals).to_h } + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + + def initialize(job_name, instance, variables, total) + @job_name = job_name + @instance = instance + @variables = variables.to_h + @total = total + end + + def attributes + { + name: name, + instance: instance, + variables: variables, + parallel: { total: total } + } + end + + def name_with_details + vars = variables.map { |key, value| "#{key}=#{value}"}.join('; ') + + "#{job_name} (#{vars})" + end + + def name + "#{job_name} #{instance}/#{total}" + end + + private + + attr_reader :job_name, :instance, :variables, :total + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer/number_strategy.rb b/lib/gitlab/ci/config/normalizer/number_strategy.rb new file mode 100644 index 00000000000..4754e7b46d4 --- /dev/null +++ b/lib/gitlab/ci/config/normalizer/number_strategy.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + class NumberStrategy + class << self + def applies_to?(config) + config.is_a?(Integer) || config.is_a?(Hash) && config.key?(:number) + end + + def build_from(job_name, config) + total = config.is_a?(Hash) ? config[:number] : config + + Array.new(total) do |index| + new(job_name, index.next, total) + end + end + end + + def initialize(job_name, instance, total) + @job_name = job_name + @instance = instance + @total = total + end + + def attributes + { + name: name, + instance: instance, + parallel: { total: total } + } + end + + def name + "#{job_name} #{instance}/#{total}" + end + + private + + attr_reader :job_name, :instance, :total + end + end + end + end +end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 6130baeb9d5..2f6667d3600 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -14,32 +14,16 @@ module Gitlab ::Feature.enabled?(:ci_job_heartbeats_runner, project, default_enabled: true) end - def self.pipeline_fixed_notifications? - ::Feature.enabled?(:ci_pipeline_fixed_notifications, default_enabled: true) - end - def self.instance_variables_ui_enabled? ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true) end - def self.composite_status?(project) - ::Feature.enabled?(:ci_composite_status, project, default_enabled: true) - end - - def self.atomic_processing?(project) - ::Feature.enabled?(:ci_atomic_processing, project, default_enabled: true) - end - def self.pipeline_latest? ::Feature.enabled?(:ci_pipeline_latest, default_enabled: true) end def self.pipeline_status_omit_commit_sha_in_cache_key?(project) - Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project) - end - - def self.release_generation_enabled? - ::Feature.enabled?(:ci_release_generation, default_enabled: true) + Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project, default_enabled: true) end # Remove in https://gitlab.com/gitlab-org/gitlab/-/issues/224199 @@ -49,13 +33,11 @@ module Gitlab # Remove in https://gitlab.com/gitlab-org/gitlab/-/issues/227052 def self.variables_api_filter_environment_scope? - ::Feature.enabled?(:ci_variables_api_filter_environment_scope, default_enabled: false) + ::Feature.enabled?(:ci_variables_api_filter_environment_scope, default_enabled: true) end - # This FF is only used for development purpose to test that warnings can be - # raised and propagated to the UI. def self.raise_job_rules_without_workflow_rules_warning? - ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning) + ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true) end def self.keep_latest_artifacts_for_ref_enabled?(project) @@ -70,8 +52,32 @@ module Gitlab ::Feature.enabled?(:ci_bulk_insert_on_create, project, default_enabled: true) end + def self.ci_if_parenthesis_enabled? + ::Feature.enabled?(:ci_if_parenthesis_enabled, default_enabled: true) + end + def self.allow_to_create_merge_request_pipelines_in_target_project?(target_project) - ::Feature.enabled?(:ci_allow_to_create_merge_request_pipelines_in_target_project, target_project) + ::Feature.enabled?(:ci_allow_to_create_merge_request_pipelines_in_target_project, target_project, default_enabled: true) + end + + def self.ci_plan_needs_size_limit?(project) + ::Feature.enabled?(:ci_plan_needs_size_limit, project, default_enabled: true) + end + + def self.job_entry_matches_all_keys? + ::Feature.enabled?(:ci_job_entry_matches_all_keys) + end + + def self.lint_creates_pipeline_with_dry_run?(project) + ::Feature.enabled?(:ci_lint_creates_pipeline_with_dry_run, project, default_enabled: true) + end + + def self.reset_ci_minutes_for_all_namespaces? + ::Feature.enabled?(:reset_ci_minutes_for_all_namespaces, default_enabled: false) + end + + def self.expand_names_for_cross_pipeline_artifacts?(project) + ::Feature.enabled?(:ci_expand_names_for_cross_pipeline_artifacts, project) end end end diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb index 006d5097148..934c797580c 100644 --- a/lib/gitlab/ci/parsers/coverage/cobertura.rb +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -28,6 +28,8 @@ module Gitlab end def parse_node(key, value, coverage_report) + return if key == 'sources' + if key == 'class' Array.wrap(value).each do |item| parse_class(item, coverage_report) diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb new file mode 100644 index 00000000000..468f3bc4689 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class CancelPendingPipelines < Chain::Base + include Chain::Helpers + + def perform! + return unless project.auto_cancel_pending_pipelines? + + Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines) do |cancelables| + cancelables.find_each do |cancelable| + cancelable.auto_cancel_running(pipeline) + end + end + end + + def break? + false + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def auto_cancelable_pipelines + project.ci_pipelines + .where(ref: pipeline.ref) + .where.not(id: pipeline.same_family_pipeline_ids) + .where.not(sha: project.commit(pipeline.ref).try(:id)) + .alive_or_scheduled + .with_only_interruptible_builds + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 74b28b181bc..dbaa6951e64 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -10,7 +10,7 @@ module Gitlab :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update, :bridge, :content, + :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, # These attributes are set by Chains during processing: :config_content, :config_processor, :stage_seeds ) do @@ -22,6 +22,8 @@ module Gitlab end end + alias_method :dry_run?, :dry_run + def branch_exists? strong_memoize(:is_branch) do project.repository.branch_exists?(ref) diff --git a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb index 3dd216b33d1..9954aedc4b7 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb @@ -12,7 +12,6 @@ module Gitlab def content strong_memoize(:content) do next unless command.content.present? - raise UnsupportedSourceError, "#{command.source} not a dangling build" unless command.dangling_build? command.content end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index aba7dab508d..d7271df1694 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -6,13 +6,13 @@ module Gitlab module Chain module Helpers def error(message, config_error: false, drop_reason: nil) - if config_error && command.save_incompleted + if config_error drop_reason = :config_error pipeline.yaml_errors = message end pipeline.add_error_message(message) - pipeline.drop!(drop_reason) if drop_reason + pipeline.drop!(drop_reason) if drop_reason && persist_pipeline? # TODO: consider not to rely on AR errors directly as they can be # polluted with other unrelated errors (e.g. state machine) @@ -23,6 +23,10 @@ module Gitlab def warning(message) pipeline.add_warning_message(message) end + + def persist_pipeline? + command.save_incompleted && !pipeline.readonly? + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb new file mode 100644 index 00000000000..0d7449813b4 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/metrics.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class Metrics < Chain::Base + def perform! + counter.increment(source: @pipeline.source) + end + + def break? + false + end + + def counter + ::Gitlab::Ci::Pipeline::Metrics.new.pipelines_created_counter + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb new file mode 100644 index 00000000000..1eb7474e915 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Pipeline + # After pipeline has been successfully created we can start processing it. + class Process < Chain::Base + def perform! + ::Ci::ProcessPipelineService + .new(@pipeline) + .execute + end + + def break? + false + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index 204c7725214..dc648568129 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -9,30 +9,21 @@ module Gitlab @pipeline = pipeline @command = command @sequence = sequence - @completed = [] @start = Time.now end def build! - @sequence.each do |chain| - step = chain.new(@pipeline, @command) + @sequence.each do |step_class| + step = step_class.new(@pipeline, @command) step.perform! break if step.break? - - @completed.push(step) end - @pipeline.tap do - yield @pipeline, self if block_given? - - @command.observe_creation_duration(Time.now - @start) - @command.observe_pipeline_size(@pipeline) - end - end + @command.observe_creation_duration(Time.now - @start) + @command.observe_pipeline_size(@pipeline) - def complete? - @completed.size == @sequence.size + @pipeline end end end diff --git a/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb b/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb new file mode 100644 index 00000000000..0e9add4ee74 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + # During the dry run we don't want to persist the pipeline and skip + # all the other steps that operate on a persisted context. + # This causes the chain to break at this point. + class StopDryRun < Chain::Base + def perform! + # no-op + end + + def break? + @command.dry_run? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 769d0dffd0b..8f1e690c081 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -34,7 +34,7 @@ module Gitlab end def allowed_to_write_ref? - access = Gitlab::UserAccess.new(current_user, project: project) + access = Gitlab::UserAccess.new(current_user, container: project) if @command.branch_exists? access.can_update_branch?(@command.ref) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/and.rb b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb index 54a0e2ad9dd..422735bd104 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/and.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class And < Lexeme::Operator + class And < Lexeme::LogicalOperator PATTERN = /&&/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb index 7ebd2e25398..676857183cf 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb @@ -10,6 +10,10 @@ module Gitlab raise NotImplementedError end + def name + self.class.name.demodulize.underscore + end + def self.build(token) raise NotImplementedError end @@ -23,6 +27,10 @@ module Gitlab def self.pattern self::PATTERN end + + def self.consume?(lexeme) + lexeme && precedence >= lexeme.precedence + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb index 62f4c14f597..d35be12c996 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class Equals < Lexeme::Operator + class Equals < Lexeme::LogicalOperator PATTERN = /==/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/logical_operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/logical_operator.rb new file mode 100644 index 00000000000..05d5043c06e --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/logical_operator.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class LogicalOperator < Lexeme::Operator + # This operator class is design to handle single operators that take two + # arguments. Expression::Parser was originally designed to read infix operators, + # and so the two operands are called "left" and "right" here. If we wish to + # implement an Operator that takes a greater or lesser number of arguments, a + # structural change or additional Operator superclass will likely be needed. + + def initialize(left, right) + raise OperatorError, 'Invalid left operand' unless left.respond_to? :evaluate + raise OperatorError, 'Invalid right operand' unless right.respond_to? :evaluate + + @left = left + @right = right + end + + def inspect + "#{name}(#{@left.inspect}, #{@right.inspect})" + end + + def self.type + :logical_operator + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index f7b0720d4a9..4d65b914d8d 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class Matches < Lexeme::Operator + class Matches < Lexeme::LogicalOperator PATTERN = /=~/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb index 8166bcd5730..64485a7e6b3 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class NotEquals < Lexeme::Operator + class NotEquals < Lexeme::LogicalOperator PATTERN = /!=/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb index 02479ed28a4..29c5aa5d753 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class NotMatches < Lexeme::Operator + class NotMatches < Lexeme::LogicalOperator PATTERN = /\!~/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb index be7258c201a..e7f7945532b 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb @@ -9,13 +9,17 @@ module Gitlab PATTERN = /null/.freeze def initialize(value = nil) - @value = nil + super end def evaluate(variables = {}) nil end + def inspect + 'null' + end + def self.build(_value) self.new end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb index 3ddab7800c8..a740c50c900 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb @@ -6,24 +6,10 @@ module Gitlab module Expression module Lexeme class Operator < Lexeme::Base - # This operator class is design to handle single operators that take two - # arguments. Expression::Parser was originally designed to read infix operators, - # and so the two operands are called "left" and "right" here. If we wish to - # implement an Operator that takes a greater or lesser number of arguments, a - # structural change or additional Operator superclass will likely be needed. - OperatorError = Class.new(Expression::ExpressionError) - def initialize(left, right) - raise OperatorError, 'Invalid left operand' unless left.respond_to? :evaluate - raise OperatorError, 'Invalid right operand' unless right.respond_to? :evaluate - - @left = left - @right = right - end - def self.type - :operator + raise NotImplementedError end def self.precedence diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/or.rb b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb index 807876f905a..c7d653ac859 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/or.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class Or < Lexeme::Operator + class Or < Lexeme::LogicalOperator PATTERN = /\|\|/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb new file mode 100644 index 00000000000..b0ca26c9f5d --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class ParenthesisClose < Lexeme::Operator + PATTERN = /\)/.freeze + + def self.type + :parenthesis_close + end + + def self.precedence + 900 + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb new file mode 100644 index 00000000000..924fe0663ab --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class ParenthesisOpen < Lexeme::Operator + PATTERN = /\(/.freeze + + def self.type + :parenthesis_open + end + + def self.precedence + # Needs to be higher than `ParenthesisClose` and all other Lexemes + 901 + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 0212fa9d661..514241e8ae2 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -11,7 +11,7 @@ module Gitlab PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze def initialize(regexp) - @value = regexp.gsub(/\\\//, '/') + super(regexp.gsub(/\\\//, '/')) unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value) raise Lexer::SyntaxError, 'Invalid regular expression!' @@ -24,6 +24,10 @@ module Gitlab raise Expression::RuntimeError, 'Invalid regular expression!' end + def inspect + "/#{value}/" + end + def self.pattern PATTERN end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index 2db2bf011f1..e90e764bcd9 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -9,13 +9,17 @@ module Gitlab PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/.freeze def initialize(value) - @value = value + super(value) end def evaluate(variables = {}) @value.to_s end + def inspect + @value.inspect + end + def self.build(string) new(string.match(PATTERN)[:string]) end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb index ef9ddb6cae9..6d872fee39d 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb @@ -9,6 +9,10 @@ module Gitlab def self.type :value end + + def initialize(value) + @value = value + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index 85c0899e4f6..11d2010909f 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -8,12 +8,12 @@ module Gitlab class Variable < Lexeme::Value PATTERN = /\$(?<name>\w+)/.freeze - def initialize(name) - @name = name + def evaluate(variables = {}) + variables.with_indifferent_access.fetch(@value, nil) end - def evaluate(variables = {}) - variables.with_indifferent_access.fetch(@name, nil) + def inspect + "$#{@value}" end def self.build(string) diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index 7d7582612f9..5b7365cb33b 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -10,6 +10,8 @@ module Gitlab SyntaxError = Class.new(Expression::ExpressionError) LEXEMES = [ + Expression::Lexeme::ParenthesisOpen, + Expression::Lexeme::ParenthesisClose, Expression::Lexeme::Variable, Expression::Lexeme::String, Expression::Lexeme::Pattern, @@ -22,6 +24,28 @@ module Gitlab Expression::Lexeme::Or ].freeze + # To be removed with `ci_if_parenthesis_enabled` + LEGACY_LEXEMES = [ + Expression::Lexeme::Variable, + Expression::Lexeme::String, + Expression::Lexeme::Pattern, + Expression::Lexeme::Null, + Expression::Lexeme::Equals, + Expression::Lexeme::Matches, + Expression::Lexeme::NotEquals, + Expression::Lexeme::NotMatches, + Expression::Lexeme::And, + Expression::Lexeme::Or + ].freeze + + def self.lexemes + if ::Gitlab::Ci::Features.ci_if_parenthesis_enabled? + LEXEMES + else + LEGACY_LEXEMES + end + end + MAX_TOKENS = 100 def initialize(statement, max_tokens: MAX_TOKENS) @@ -47,7 +71,7 @@ module Gitlab return tokens if @scanner.eos? - lexeme = LEXEMES.find do |type| + lexeme = self.class.lexemes.find do |type| type.scan(@scanner).tap do |token| tokens.push(token) if token.present? end diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index edb55edf356..27d7aa2f37e 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -15,11 +15,18 @@ module Gitlab def tree results = [] - tokens_rpn.each do |token| + tokens = + if ::Gitlab::Ci::Features.ci_if_parenthesis_enabled? + tokens_rpn + else + legacy_tokens_rpn + end + + tokens.each do |token| case token.type when :value results.push(token.build) - when :operator + when :logical_operator right_operand = results.pop left_operand = results.pop @@ -27,7 +34,7 @@ module Gitlab results.push(res) end else - raise ParseError, 'Unprocessable token found in parse tree' + raise ParseError, "Unprocessable token found in parse tree: #{token.type}" end end @@ -45,6 +52,7 @@ module Gitlab # Parse the expression into Reverse Polish Notation # (See: Shunting-yard algorithm) + # Taken from: https://en.wikipedia.org/wiki/Shunting-yard_algorithm#The_algorithm_in_detail def tokens_rpn output = [] operators = [] @@ -53,7 +61,34 @@ module Gitlab case token.type when :value output.push(token) - when :operator + when :logical_operator + output.push(operators.pop) while token.lexeme.consume?(operators.last&.lexeme) + + operators.push(token) + when :parenthesis_open + operators.push(token) + when :parenthesis_close + output.push(operators.pop) while token.lexeme.consume?(operators.last&.lexeme) + + raise ParseError, 'Unmatched parenthesis' unless operators.last + + operators.pop if operators.last.lexeme.type == :parenthesis_open + end + end + + output.concat(operators.reverse) + end + + # To be removed with `ci_if_parenthesis_enabled` + def legacy_tokens_rpn + output = [] + operators = [] + + @tokens.each do |token| + case token.type + when :value + output.push(token) + when :logical_operator if operators.any? && token.lexeme.precedence >= operators.last.lexeme.precedence output.push(operators.pop) end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 649da745eea..db6cca27f1c 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -36,6 +36,15 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end end + + def pipelines_created_counter + strong_memoize(:pipelines_created_count) do + name = :pipelines_created_total + comment = 'Counter of pipelines created' + + Gitlab::Metrics.counter(name, comment) + end + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 114a46ca9f6..3be3fa63b92 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,9 +11,7 @@ module Gitlab delegate :dig, to: :@seed_attributes - # When the `ci_dag_limit_needs` is enabled it uses the lower limit - LOW_NEEDS_LIMIT = 10 - HARD_NEEDS_LIMIT = 50 + DEFAULT_NEEDS_LIMIT = 10 def initialize(pipeline, attributes, previous_stages) @pipeline = pipeline @@ -142,10 +140,10 @@ module Gitlab end def max_needs_allowed - if Feature.enabled?(:ci_dag_limit_needs, @project, default_enabled: true) - LOW_NEEDS_LIMIT + if ::Gitlab::Ci::Features.ci_plan_needs_size_limit?(@pipeline.project) + @pipeline.project.actual_limits.ci_needs_size_limit else - HARD_NEEDS_LIMIT + DEFAULT_NEEDS_LIMIT end end diff --git a/lib/gitlab/ci/reports/accessibility_reports_comparer.rb b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb index fa6337166d5..210eb17f2d3 100644 --- a/lib/gitlab/ci/reports/accessibility_reports_comparer.rb +++ b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb @@ -17,7 +17,7 @@ module Gitlab end def status - head_reports.errors_count.positive? ? STATUS_FAILED : STATUS_SUCCESS + head_reports.errors_count > 0 ? STATUS_FAILED : STATUS_SUCCESS end def existing_errors diff --git a/lib/gitlab/ci/reports/test_report_summary.rb b/lib/gitlab/ci/reports/test_report_summary.rb index 85b83b790e7..3e7227b7223 100644 --- a/lib/gitlab/ci/reports/test_report_summary.rb +++ b/lib/gitlab/ci/reports/test_report_summary.rb @@ -4,42 +4,17 @@ module Gitlab module Ci module Reports class TestReportSummary - attr_reader :all_results - - def initialize(all_results) - @all_results = all_results + def initialize(build_report_results) + @build_report_results = build_report_results + @suite_summary = TestSuiteSummary.new(@build_report_results) end def total - TestSuiteSummary.new(all_results) - end - - def total_time - total.total_time - end - - def total_count - total.total_count - end - - def success_count - total.success_count - end - - def failed_count - total.failed_count - end - - def skipped_count - total.skipped_count - end - - def error_count - total.error_count + @suite_summary.to_h end def test_suites - all_results + @build_report_results .group_by(&:tests_name) .transform_values { |results| TestSuiteSummary.new(results) } end diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb index 86ba725c71e..a5a630642e5 100644 --- a/lib/gitlab/ci/reports/test_reports.rb +++ b/lib/gitlab/ci/reports/test_reports.rb @@ -43,9 +43,7 @@ module Gitlab end def suite_errors - test_suites.each_with_object({}) do |(name, suite), errors| - errors[suite.name] = suite.suite_error if suite.suite_error - end + test_suites.transform_values(&:suite_error).compact end TestCase::STATUS_TYPES.each do |status_type| diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index 28b81e7a471..5ee779227ec 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -28,7 +28,7 @@ module Gitlab def total_count return 0 if suite_error - test_cases.values.sum(&:count) + [success_count, failed_count, skipped_count, error_count].sum end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/reports/test_suite_summary.rb b/lib/gitlab/ci/reports/test_suite_summary.rb index f9b0bedb712..32b06d0ad49 100644 --- a/lib/gitlab/ci/reports/test_suite_summary.rb +++ b/lib/gitlab/ci/reports/test_suite_summary.rb @@ -4,45 +4,54 @@ module Gitlab module Ci module Reports class TestSuiteSummary - attr_reader :results - - def initialize(results) - @results = results + def initialize(build_report_results) + @build_report_results = build_report_results end def name - @name ||= results.first.tests_name + @name ||= @build_report_results.first.tests_name end # rubocop: disable CodeReuse/ActiveRecord def build_ids - results.pluck(:build_id) + @build_report_results.pluck(:build_id) end def total_time - @total_time ||= results.sum(&:tests_duration) + @total_time ||= @build_report_results.sum(&:tests_duration) end def success_count - @success_count ||= results.sum(&:tests_success) + @success_count ||= @build_report_results.sum(&:tests_success) end def failed_count - @failed_count ||= results.sum(&:tests_failed) + @failed_count ||= @build_report_results.sum(&:tests_failed) end def skipped_count - @skipped_count ||= results.sum(&:tests_skipped) + @skipped_count ||= @build_report_results.sum(&:tests_skipped) end def error_count - @error_count ||= results.sum(&:tests_errored) + @error_count ||= @build_report_results.sum(&:tests_errored) end def total_count @total_count ||= [success_count, failed_count, skipped_count, error_count].sum end # rubocop: disable CodeReuse/ActiveRecord + + def to_h + { + time: total_time, + count: total_count, + success: success_count, + failed: failed_count, + skipped: skipped_count, + error: error_count + } + end end end end diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb new file mode 100644 index 00000000000..2171637687f --- /dev/null +++ b/lib/gitlab/ci/runner_instructions.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class RunnerInstructions + class ArgumentError < ::ArgumentError; end + + include Gitlab::Allowable + + OS = { + linux: { + human_readable_name: "Linux", + download_locations: { + amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-amd64", + '386': "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-386", + arm: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-arm", + arm64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-arm64" + }, + install_script_template_path: "lib/gitlab/ci/runner_instructions/templates/linux/install.sh", + runner_executable: "sudo gitlab-runner" + }, + osx: { + human_readable_name: "macOS", + download_locations: { + amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-darwin-amd64" + }, + install_script_template_path: "lib/gitlab/ci/runner_instructions/templates/osx/install.sh", + runner_executable: "sudo gitlab-runner" + }, + windows: { + human_readable_name: "Windows", + download_locations: { + amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-windows-amd64.exe", + '386': "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-windows-386.exe" + }, + install_script_template_path: "lib/gitlab/ci/runner_instructions/templates/windows/install.ps1", + runner_executable: "./gitlab-runner.exe" + } + }.freeze + + OTHER_ENVIRONMENTS = { + docker: { + human_readable_name: "Docker", + installation_instructions_url: "https://docs.gitlab.com/runner/install/docker.html" + }, + kubernetes: { + human_readable_name: "Kubernetes", + installation_instructions_url: "https://docs.gitlab.com/runner/install/kubernetes.html" + } + }.freeze + + attr_reader :errors + + def initialize(current_user:, group: nil, project: nil, os:, arch:) + @current_user = current_user + @group = group + @project = project + @os = os + @arch = arch + @errors = [] + + validate_params + end + + def install_script + with_error_handling [Gitlab::Ci::RunnerInstructions::ArgumentError] do + raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('Architecture not found for OS') unless environment[:download_locations].key?(@arch.to_sym) + + replace_variables(get_file(environment[:install_script_template_path])) + end + end + + def register_command + with_error_handling [Gitlab::Ci::RunnerInstructions::ArgumentError, Gitlab::Access::AccessDeniedError] do + raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('No runner executable') unless environment[:runner_executable] + + server_url = Gitlab::Routing.url_helpers.root_url(only_path: false) + runner_executable = environment[:runner_executable] + + "#{runner_executable} register --url #{server_url} --registration-token #{registration_token}" + end + end + + private + + def with_error_handling(exceptions) + return if errors.present? + + yield + rescue *exceptions => e + @errors << e.message + nil + end + + def environment + @environment ||= OS[@os.to_sym] || ( raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('Invalid OS') ) + end + + def validate_params + @errors << s_('Missing OS') unless @os.present? + @errors << s_('Missing arch') unless @arch.present? + end + + def replace_variables(expression) + expression.sub('${GITLAB_CI_RUNNER_DOWNLOAD_LOCATION}', "#{environment[:download_locations][@arch.to_sym]}") + end + + def get_file(path) + File.read(path) + end + + def registration_token + project_token || group_token || instance_token + end + + def project_token + return unless @project + raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_pipeline, @project) + + @project.runners_token + end + + def group_token + return unless @group + raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_group, @group) + + @group.runners_token + end + + def instance_token + raise Gitlab::Access::AccessDeniedError unless @current_user&.admin? + + Gitlab::CurrentSettings.runners_registration_token + end + end + end +end diff --git a/lib/gitlab/ci/runner_instructions/templates/linux/install.sh b/lib/gitlab/ci/runner_instructions/templates/linux/install.sh new file mode 100644 index 00000000000..6c8a0796d23 --- /dev/null +++ b/lib/gitlab/ci/runner_instructions/templates/linux/install.sh @@ -0,0 +1,12 @@ +# Download the binary for your system +sudo curl -L --output /usr/local/bin/gitlab-runner ${GITLAB_CI_RUNNER_DOWNLOAD_LOCATION} + +# Give it permissions to execute +sudo chmod +x /usr/local/bin/gitlab-runner + +# Create a GitLab CI user +sudo useradd --comment 'GitLab Runner' --create-home gitlab-runner --shell /bin/bash + +# Install and run as service +sudo gitlab-runner install --user=gitlab-runner --working-directory=/home/gitlab-runner +sudo gitlab-runner start diff --git a/lib/gitlab/ci/runner_instructions/templates/osx/install.sh b/lib/gitlab/ci/runner_instructions/templates/osx/install.sh new file mode 100644 index 00000000000..de4ee3e52fc --- /dev/null +++ b/lib/gitlab/ci/runner_instructions/templates/osx/install.sh @@ -0,0 +1,11 @@ +# Download the binary for your system +sudo curl --output /usr/local/bin/gitlab-runner ${GITLAB_CI_RUNNER_DOWNLOAD_LOCATION} + +# Give it permissions to execute +sudo chmod +x /usr/local/bin/gitlab-runner + +# The rest of commands execute as the user who will run the Runner +# Register the Runner (steps below), then run +cd ~ +gitlab-runner install +gitlab-runner start diff --git a/lib/gitlab/ci/runner_instructions/templates/windows/install.ps1 b/lib/gitlab/ci/runner_instructions/templates/windows/install.ps1 new file mode 100644 index 00000000000..dc37f88543c --- /dev/null +++ b/lib/gitlab/ci/runner_instructions/templates/windows/install.ps1 @@ -0,0 +1,13 @@ +# Run PowerShell: https://docs.microsoft.com/en-us/powershell/scripting/windows-powershell/starting-windows-powershell?view=powershell-7#with-administrative-privileges-run-as-administrator +# Create a folder somewhere in your system ex.: C:\GitLab-Runner +New-Item -Path 'C:\GitLab-Runner' -ItemType Directory + +# Enter the folder +cd 'C:\GitLab-Runner' + +# Dowload binary +Invoke-WebRequest -Uri "${GITLAB_CI_RUNNER_DOWNLOAD_LOCATION}" -OutFile "gitlab-runner.exe" + +# Register the Runner (steps below), then run +.\gitlab-runner.exe install +.\gitlab-runner.exe start diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 76ad113aad9..88846f724e7 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -24,7 +24,8 @@ module Gitlab downstream_bridge_project_not_found: 'downstream project could not be found', insufficient_bridge_permissions: 'no permissions to trigger downstream pipeline', bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline', - downstream_pipeline_creation_failed: 'downstream pipeline can not be created' + downstream_pipeline_creation_failed: 'downstream pipeline can not be created', + secrets_provider_not_found: 'secrets provider can not be found' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index c10d87a537b..968ff0fce89 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -162,4 +162,4 @@ include: - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml - - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml + - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml index 5f4bd631db6..c1815baf7e6 100644 --- a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml @@ -1,4 +1,4 @@ -# This template is deprecated and will be removed as part of GitLab 13.2! +# This template is deprecated. # # If you have referenced this template in your CI pipeline, please # update your CI configuration by replacing the following occurrence(s): @@ -20,12 +20,8 @@ stages: - deploy - production -before_script: - - printf '\nWARNING!\nThis job includes "Deploy-ECS.gitlab-ci.yml". Please rename this to "AWS/Deploy-ECS.gitlab-ci.yml".\n' - -variables: - AUTO_DEVOPS_PLATFORM_TARGET: ECS - -include: - - template: Jobs/Build.gitlab-ci.yml - - template: Jobs/Deploy/ECS.gitlab-ci.yml +"error: Template has moved": + stage: deploy + script: + - echo "Deploy-ECS.gitlab-ci.yml has been moved to AWS/Deploy-ECS.gitlab-ci.yml, see https://docs.gitlab.com/ee/ci/cloud_deployment/#deploy-your-application-to-the-aws-elastic-container-service-ecs for more details." + - exit 1 diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index dbe870953ae..0c3598a61a7 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,6 +1,6 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.3.1" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.4.0" variables: DOCKER_TLS_CERTDIR: "" services: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 6b76d7e0c9b..cf851c875ee 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10-gitlab.1" needs: [] script: - | diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index d7d927ac8ee..f234008dad4 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.2" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.0" dast_environment_deploy: extends: .dast-auto-deploy @@ -23,7 +23,7 @@ dast_environment_deploy: when: never - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH when: never - - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given + - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given when: never - if: $CI_COMMIT_BRANCH && $CI_KUBERNETES_ACTIVE && @@ -46,7 +46,7 @@ stop_dast_environment: when: never - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH when: never - - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given + - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given when: never - if: $CI_COMMIT_BRANCH && $CI_KUBERNETES_ACTIVE && diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 66c60e85892..76fb2948144 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.2" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.0" dependencies: [] include: diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml index b437ddbd734..4a9849c85c9 100644 --- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml @@ -5,7 +5,7 @@ load_performance: variables: DOCKER_TLS_CERTDIR: "" K6_IMAGE: loadimpact/k6 - K6_VERSION: 0.26.2 + K6_VERSION: 0.27.0 K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js K6_OPTIONS: '' services: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml new file mode 100644 index 00000000000..e87f0f28d01 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -0,0 +1,146 @@ +stages: + - build + - test + - deploy + - fuzz + +variables: + FUZZAPI_PROFILE: Quick + FUZZAPI_VERSION: latest + FUZZAPI_CONFIG: "/app/.gitlab-api-fuzzing.yml" + FUZZAPI_TIMEOUT: 30 + FUZZAPI_REPORT: gl-api-fuzzing-report.xml + # + FUZZAPI_D_NETWORK: testing-net + # + # Wait up to 5 minutes for API Fuzzer and target url to become + # available (non 500 response to HTTP(s)) + FUZZAPI_SERVICE_START_TIMEOUT: "300" + # + +apifuzzer_fuzz: + stage: fuzz + image: docker:19.03.12 + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + FUZZAPI_PROJECT: $CI_PROJECT_PATH + FUZZAPI_API: http://apifuzzer:80 + allow_failure: true + rules: + - if: $API_FUZZING_DISABLED + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $FUZZAPI_HAR == null && + $FUZZAPI_OPENAPI == null && + $FUZZAPI_D_WORKER_IMAGE == null + when: never + - if: $FUZZAPI_D_WORKER_IMAGE == null && + $FUZZAPI_TARGET_URL == null + when: never + - if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + services: + - docker:19.03.12-dind + script: + # + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + # + - docker network create --driver bridge $FUZZAPI_D_NETWORK + # + # Run user provided pre-script + - sh -c "$FUZZAPI_PRE_SCRIPT" + # + # Start peach testing engine container + - | + docker run -d \ + --name apifuzzer \ + --network $FUZZAPI_D_NETWORK \ + -e Proxy:Port=8000 \ + -e TZ=America/Los_Angeles \ + -e FUZZAPI_API=http://127.0.0.1:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e GITLAB_FEATURES \ + -v $CI_PROJECT_DIR:/app \ + -p 80:80 \ + -p 8000:8000 \ + -p 514:514 \ + --restart=no \ + registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing-src:${FUZZAPI_VERSION}-engine + # + # Start target container + - | + if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \ + docker run -d \ + --name target \ + --network $FUZZAPI_D_NETWORK \ + $FUZZAPI_D_TARGET_ENV \ + $FUZZAPI_D_TARGET_PORTS \ + $FUZZAPI_D_TARGET_VOLUME \ + --restart=no \ + $FUZZAPI_D_TARGET_IMAGE \ + ; fi + # + # Start worker container + - | + if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ + echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE" \ + docker run \ + --name worker \ + --network $FUZZAPI_D_NETWORK \ + -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_AUTOMATION_CMD \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ + $FUZZAPI_D_WORKER_ENV \ + $FUZZAPI_D_WORKER_PORTS \ + $FUZZAPI_D_WORKER_VOLUME \ + --restart=no \ + $FUZZAPI_D_WORKER_IMAGE \ + ; fi + # + # Wait for testing to complete if api fuzzer is scanning + - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI" != "" ]; then echo "Waiting for API Fuzzer to exit"; docker wait apifuzzer; fi + # + # Run user provided pre-script + - sh -c "$FUZZAPI_POST_SCRIPT" + # + after_script: + # + # Shutdown all containers + - echo "Stopping all containers" + - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi + - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker stop worker; fi + - docker stop apifuzzer + # + # Save docker logs + - docker logs apifuzzer &> gl-api_fuzzing-logs.log + - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi + - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker logs worker &> gl-api_fuzzing-worker-logs.log; fi + # + artifacts: + when: always + paths: + - ./gl-api_fuzzing*.log + - ./gl-api_fuzzing*.zip + reports: + junit: $FUZZAPI_REPORT + +# end diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 2fab8b95a3d..3f47e575afd 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -3,22 +3,26 @@ variables: # Which branch we want to run full fledged long running fuzzing jobs. # All others will run fuzzing regression - COVERAGE_FUZZING_BRANCH: "$CI_DEFAULT_BRANCH" - # This is using semantic version and will always download latest v1 gitlab-cov-fuzz release - COVERAGE_FUZZING_VERSION: v1 + COVFUZZ_BRANCH: "$CI_DEFAULT_BRANCH" + # This is using semantic version and will always download latest v2 gitlab-cov-fuzz release + COVFUZZ_VERSION: v2 # This is for users who have an offline environment and will have to replicate gitlab-cov-fuzz release binaries # to their own servers - COVERAGE_FUZZING_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw" + COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw" + .fuzz_base: stage: fuzz allow_failure: true before_script: + - export COVFUZZ_JOB_TOKEN=$CI_JOB_TOKEN + - export COVFUZZ_PRIVATE_TOKEN=$CI_PRIVATE_TOKEN + - export COVFUZZ_PROJECT_ID=$CI_PROJECT_ID - if [ -x "$(command -v apt-get)" ] ; then apt-get update && apt-get install -y wget; fi - - wget -O gitlab-cov-fuzz "${COVERAGE_FUZZING_URL_PREFIX}"/"${COVERAGE_FUZZING_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64 + - wget -O gitlab-cov-fuzz "${COVFUZZ_URL_PREFIX}"/"${COVFUZZ_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64 - chmod a+x gitlab-cov-fuzz - export REGRESSION=true - - if [[ $CI_COMMIT_BRANCH = $COVERAGE_FUZZING_BRANCH ]]; then REGRESSION=false; fi; + - if [[ $CI_COMMIT_BRANCH = $COVFUZZ_BRANCH ]]; then REGRESSION=false; fi; artifacts: paths: - corpus @@ -28,7 +32,7 @@ variables: coverage_fuzzing: gl-coverage-fuzzing-report.json when: always rules: - - if: $COVERAGE_FUZZING_DISABLED + - if: $COVFUZZ_DISABLED when: never - if: $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/ - if: $CI_RUNNER_EXECUTABLE_ARCH == "linux" diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 37f6cd216ca..d5275c57ef8 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -111,6 +111,7 @@ gemnasium-dependency_scanning: - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' + - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' gemnasium-maven-dependency_scanning: extends: .ds-analyzer @@ -144,8 +145,8 @@ gemnasium-python-dependency_scanning: - '{Pipfile,*/Pipfile,*/*/Pipfile}' - '{requires.txt,*/requires.txt,*/*/requires.txt}' - '{setup.py,*/setup.py,*/*/setup.py}' - # Support passing of $PIP_REQUIREMENTS_FILE - # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + # Support passing of $PIP_REQUIREMENTS_FILE + # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ && diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index f0e2f48dd5c..6eb17341472 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -52,8 +52,7 @@ sast: rules: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ + - if: $CI_COMMIT_BRANCH script: - /analyzer run @@ -65,7 +64,6 @@ bandit-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /bandit/ exists: - '**/*.py' @@ -106,7 +104,6 @@ flawfinder-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ exists: - '**/*.c' @@ -120,7 +117,6 @@ kubesec-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && $SCAN_KUBERNETES_MANIFESTS == 'true' @@ -132,7 +128,6 @@ gosec-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /gosec/ exists: - '**/*.go' @@ -145,7 +140,6 @@ nodejs-scan-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ exists: - 'package.json' @@ -158,7 +152,6 @@ phpcs-security-audit-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ exists: - '**/*.php' @@ -171,7 +164,6 @@ pmd-apex-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ exists: - '**/*.cls' @@ -184,7 +176,6 @@ secrets-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /secrets/ security-code-scan-sast: @@ -195,7 +186,6 @@ security-code-scan-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ exists: - '**/*.csproj' @@ -209,7 +199,6 @@ sobelow-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /sobelow/ exists: - 'mix.exs' @@ -222,7 +211,6 @@ spotbugs-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ exists: - '**/*.groovy' diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index 441a57048e1..b897c7b482f 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -7,6 +7,8 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" SECRETS_ANALYZER_VERSION: "3" + SECRET_DETECTION_EXCLUDED_PATHS: "" + .secret-analyzer: stage: test @@ -21,8 +23,7 @@ secret_detection_default_branch: rules: - if: $SECRET_DETECTION_DISABLED when: never - - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsecret_detection\b/ + - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH script: - /analyzer run @@ -31,8 +32,7 @@ secret_detection: rules: - if: $SECRET_DETECTION_DISABLED when: never - - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && - $GITLAB_FEATURES =~ /\bsecret_detection\b/ + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH script: - git fetch origin $CI_DEFAULT_BRANCH $CI_BUILD_REF_NAME - export SECRET_DETECTION_COMMIT_TO=$(git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_BUILD_REF_NAME | tail -n 1) diff --git a/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml index 77a1b57d92f..584e6966180 100644 --- a/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml @@ -1,4 +1,5 @@ rspec-rails-modified-path-specs: + image: ruby:2.6 stage: .pre rules: - if: $CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "merge_train" diff --git a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml index d39bd234020..f964b3b2caf 100644 --- a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml @@ -11,7 +11,7 @@ load_performance: image: docker:git variables: K6_IMAGE: loadimpact/k6 - K6_VERSION: 0.26.2 + K6_VERSION: 0.27.0 K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js K6_OPTIONS: '' services: diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 8cf355bbfc1..b7046064f44 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -115,7 +115,7 @@ module Gitlab end def release(job) - job[:release] if Gitlab::Ci::Features.release_generation_enabled? + job[:release] end def stage_builds_attributes(stage) diff --git a/lib/gitlab/config/entry/legacy_validation_helpers.rb b/lib/gitlab/config/entry/legacy_validation_helpers.rb index 0a629075302..415f6f77214 100644 --- a/lib/gitlab/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/config/entry/legacy_validation_helpers.rb @@ -6,17 +6,27 @@ module Gitlab module LegacyValidationHelpers private - def validate_duration(value) - value.is_a?(String) && ChronicDuration.parse(value) + def validate_duration(value, parser = nil) + return false unless value.is_a?(String) + + if parser && parser.respond_to?(:validate_duration) + parser.validate_duration(value) + else + ChronicDuration.parse(value) + end rescue ChronicDuration::DurationParseError false end - def validate_duration_limit(value, limit) + def validate_duration_limit(value, limit, parser = nil) return false unless value.is_a?(String) - ChronicDuration.parse(value).second.from_now < - ChronicDuration.parse(limit).second.from_now + if parser && parser.respond_to?(:validate_duration_limit) + parser.validate_duration_limit(value, limit) + else + ChronicDuration.parse(value).second.from_now < + ChronicDuration.parse(limit).second.from_now + end rescue ChronicDuration::DurationParseError false end @@ -30,10 +40,18 @@ module Gitlab end def validate_variables(variables) + variables.is_a?(Hash) && variables.flatten.all?(&method(:validate_alphanumeric)) + end + + def validate_array_value_variables(variables) variables.is_a?(Hash) && - variables.flatten.all? do |value| - validate_string(value) || validate_integer(value) - end + variables.keys.all?(&method(:validate_alphanumeric)) && + variables.values.all?(&:present?) && + variables.values.flatten(1).all?(&method(:validate_alphanumeric)) + end + + def validate_alphanumeric(value) + validate_string(value) || validate_integer(value) end def validate_integer(value) diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index d1c23c41d35..a7ec98ace6e 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -106,12 +106,12 @@ module Gitlab include LegacyValidationHelpers def validate_each(record, attribute, value) - unless validate_duration(value) + unless validate_duration(value, options[:parser]) record.errors.add(attribute, 'should be a duration') end if options[:limit] - unless validate_duration_limit(value, options[:limit]) + unless validate_duration_limit(value, options[:limit], options[:parser]) record.errors.add(attribute, 'should not exceed the limit') end end @@ -272,10 +272,24 @@ module Gitlab include LegacyValidationHelpers def validate_each(record, attribute, value) + if options[:array_values] + validate_key_array_values(record, attribute, value) + else + validate_key_values(record, attribute, value) + end + end + + def validate_key_values(record, attribute, value) unless validate_variables(value) record.errors.add(attribute, 'should be a hash of key value pairs') end end + + def validate_key_array_values(record, attribute, value) + unless validate_array_value_variables(value) + record.errors.add(attribute, 'should be a hash of key value pairs, value can be an array') + end + end end class ExpressionValidator < ActiveModel::EachValidator diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb index af828acb9c0..dfcdbdf39e0 100644 --- a/lib/gitlab/config_checker/external_database_checker.rb +++ b/lib/gitlab/config_checker/external_database_checker.rb @@ -9,35 +9,41 @@ module Gitlab notices = [] unless Gitlab::Database.postgresql_minimum_supported_version? + string_args = { + pg_version_current: Gitlab::Database.version, + pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, + pg_requirements_url_open: '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">'.html_safe, + pg_requirements_url_close: '</a>'.html_safe + } + notices << { type: 'warning', - message: _('You are using PostgreSQL %{pg_version_current}, but PostgreSQL ' \ + message: html_escape(_('You are using PostgreSQL %{pg_version_current}, but PostgreSQL ' \ '%{pg_version_minimum} is required for this version of GitLab. ' \ 'Please upgrade your environment to a supported PostgreSQL version, ' \ - 'see %{pg_requirements_url} for details.') % { - pg_version_current: Gitlab::Database.version, - pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, - pg_requirements_url: '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">database requirements</a>' - } + 'see %{pg_requirements_url_open}database requirements%{pg_requirements_url_close} for details.')) % string_args } end if Gitlab::Database.postgresql_upcoming_deprecation? && Gitlab::Database.within_deprecation_notice_window? upcoming_deprecation = Gitlab::Database::UPCOMING_POSTGRES_VERSION_DETAILS + string_args = { + pg_version_upcoming: upcoming_deprecation[:pg_version_minimum], + gl_version_upcoming: upcoming_deprecation[:gl_version], + gl_version_upcoming_date: upcoming_deprecation[:gl_version_date], + pg_version_upcoming_url_open: "<a href=\"#{upcoming_deprecation[:url]}\">".html_safe, + pg_version_upcoming_url_close: '</a>'.html_safe + } + notices << { type: 'warning', - message: _('Note that PostgreSQL %{pg_version_upcoming} will become the minimum required ' \ + message: html_escape(_('Note that PostgreSQL %{pg_version_upcoming} will become the minimum required ' \ 'version in GitLab %{gl_version_upcoming} (%{gl_version_upcoming_date}). Please ' \ 'consider upgrading your environment to a supported PostgreSQL version soon, ' \ - 'see <a href="%{pg_version_upcoming_url}">the related epic</a> for details.') % { - pg_version_upcoming: upcoming_deprecation[:pg_version_minimum], - gl_version_upcoming: upcoming_deprecation[:gl_version], - gl_version_upcoming_date: upcoming_deprecation[:gl_version_date], - pg_version_upcoming_url: upcoming_deprecation[:url] - } + 'see %{pg_version_upcoming_url_open}the related epic%{pg_version_upcoming_url_close} for details.')) % string_args } end diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index 07ae430c45e..6c6dd90e450 100644 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -6,7 +6,7 @@ module Gitlab include BaseQuery include GroupProjectsProvider - attr_reader :projections, :query, :stage, :order, :options + attr_reader :projections, :query, :stage, :options MAX_EVENTS = 50 diff --git a/lib/gitlab/cycle_analytics/summary/value.rb b/lib/gitlab/cycle_analytics/summary/value.rb index e443e037517..36306fa7c45 100644 --- a/lib/gitlab/cycle_analytics/summary/value.rb +++ b/lib/gitlab/cycle_analytics/summary/value.rb @@ -34,7 +34,7 @@ module Gitlab end def to_s - value.zero? ? '0' : value.to_s + value == 0 ? '0' : value.to_s end def to_i diff --git a/lib/gitlab/cycle_analytics/summary_helper.rb b/lib/gitlab/cycle_analytics/summary_helper.rb index 3cf9f463024..11e48679a40 100644 --- a/lib/gitlab/cycle_analytics/summary_helper.rb +++ b/lib/gitlab/cycle_analytics/summary_helper.rb @@ -4,7 +4,7 @@ module Gitlab module CycleAnalytics module SummaryHelper def frequency(count, from, to) - return Summary::Value::None.new if count.zero? + return Summary::Value::None.new if count == 0 freq = (count / days(from, to)).round(1) diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index db799c094b2..077c71f1233 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -53,7 +53,7 @@ module Gitlab def ee? # Support former project name for `dev` and support local Danger run - %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?('../../ee') + %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?(File.expand_path('../../../ee', __dir__)) end def gitlab_helper @@ -124,7 +124,7 @@ module Gitlab }.freeze # First-match win, so be sure to put more specific regex at the top... CATEGORIES = { - [%r{usage_data}, %r{^(\+|-).*(count|distinct_count)\(.*\)(.*)$}] => [:database, :backend], + [%r{usage_data\.rb}, %r{^(\+|-).*(count|distinct_count)\(.*\)(.*)$}] => [:database, :backend], %r{\Adoc/.*(\.(md|png|gif|jpg))\z} => :docs, %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs, @@ -170,10 +170,15 @@ module Gitlab %r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity, %r{\A(ee/)?scripts/} => :engineering_productivity, %r{\Atooling/} => :engineering_productivity, + %r{(CODEOWNERS)} => :engineering_productivity, + + %r{\A(ee/)?spec/features/} => :test, + %r{\A(ee/)?spec/support/shared_examples/features/} => :test, + %r{\A(ee/)?spec/support/shared_contexts/features/} => :test, + %r{\A(ee/)?spec/support/helpers/features/} => :test, %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, %r{\A(ee/)?(bin|config|generator_templates|lib|rubocop)/} => :backend, - %r{\A(ee/)?spec/features/} => :test, %r{\A(ee/)?spec/} => :backend, %r{\A(ee/)?vendor/} => :backend, %r{\A(Gemfile|Gemfile.lock|Rakefile)\z} => :backend, @@ -249,6 +254,10 @@ module Gitlab "/label #{labels_list(labels, sep: ' ')}" end + def changed_files(regex) + all_changed_files.grep(regex) + end + private def has_database_scoped_labels?(current_mr_labels) diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb index ed4af3f4a43..2e6181d1cab 100644 --- a/lib/gitlab/danger/roulette.rb +++ b/lib/gitlab/danger/roulette.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative 'teammate' +require_relative 'request_helper' module Gitlab module Danger @@ -12,45 +13,49 @@ module Gitlab database: false }.freeze - Spin = Struct.new(:category, :reviewer, :maintainer, :optional_role) + Spin = Struct.new(:category, :reviewer, :maintainer, :optional_role, :timezone_experiment) + + def team_mr_author + team.find { |person| person.username == mr_author_username } + end # Assigns GitLab team members to be reviewer and maintainer # for each change category that a Merge Request contains. # # @return [Array<Spin>] - def spin(project, categories, branch_name, timezone_experiment: false) - team = - begin - project_team(project) - rescue => err - warn("Reviewer roulette failed to load team data: #{err.message}") - [] - end - - canonical_branch_name = canonical_branch_name(branch_name) - - spin_per_category = categories.each_with_object({}) do |category, memo| + def spin(project, categories, timezone_experiment: false) + spins = categories.map do |category| including_timezone = INCLUDE_TIMEZONE_FOR_CATEGORY.fetch(category, timezone_experiment) - memo[category] = spin_for_category(team, project, category, canonical_branch_name, timezone_experiment: including_timezone) + spin_for_category(project, category, timezone_experiment: including_timezone) end - spin_per_category.map do |category, spin| - case category + backend_spin = spins.find { |spin| spin.category == :backend } + + spins.each do |spin| + including_timezone = INCLUDE_TIMEZONE_FOR_CATEGORY.fetch(spin.category, timezone_experiment) + case spin.category + when :qa + # MR includes QA changes, but also other changes, and author isn't an SET + if categories.size > 1 && !team_mr_author&.reviewer?(project, spin.category, []) + spin.optional_role = :maintainer + end when :test + spin.optional_role = :maintainer + if spin.reviewer.nil? # Fetch an already picked backend reviewer, or pick one otherwise - spin.reviewer = spin_per_category[:backend]&.reviewer || spin_for_category(team, project, :backend, canonical_branch_name).reviewer + spin.reviewer = backend_spin&.reviewer || spin_for_category(project, :backend, timezone_experiment: including_timezone).reviewer end when :engineering_productivity if spin.maintainer.nil? # Fetch an already picked backend maintainer, or pick one otherwise - spin.maintainer = spin_per_category[:backend]&.maintainer || spin_for_category(team, project, :backend, canonical_branch_name).maintainer + spin.maintainer = backend_spin&.maintainer || spin_for_category(project, :backend, timezone_experiment: including_timezone).maintainer end end - - spin end + + spins end # Looks up the current list of GitLab team members and parses it into a @@ -73,14 +78,9 @@ module Gitlab # @return [Array<Teammate>] def project_team(project_name) team.select { |member| member.in_project?(project_name) } - end - - def canonical_branch_name(branch_name) - branch_name.gsub(/^[ce]e-|-[ce]e$/, '') - end - - def new_random(seed) - Random.new(Digest::MD5.hexdigest(seed).to_i(16)) + rescue => err + warn("Reviewer roulette failed to load team data: #{err.message}") + [] end # Known issue: If someone is rejected due to OOO, and then becomes not OOO, the @@ -113,16 +113,35 @@ module Gitlab # @param [Teammate] person # @return [Boolean] def mr_author?(person) - person.username == gitlab.mr_author + person.username == mr_author_username + end + + def mr_author_username + helper.gitlab_helper&.mr_author || `whoami` + end + + def mr_source_branch + return `git rev-parse --abbrev-ref HEAD` unless helper.gitlab_helper&.mr_json + + helper.gitlab_helper.mr_json['source_branch'] + end + + def mr_labels + helper.gitlab_helper&.mr_labels || [] + end + + def new_random(seed) + Random.new(Digest::MD5.hexdigest(seed).to_i(16)) end def spin_role_for_category(team, role, project, category) team.select do |member| - member.public_send("#{role}?", project, category, gitlab.mr_labels) # rubocop:disable GitlabSecurity/PublicSend + member.public_send("#{role}?", project, category, mr_labels) # rubocop:disable GitlabSecurity/PublicSend end end - def spin_for_category(team, project, category, branch_name, timezone_experiment: false) + def spin_for_category(project, category, timezone_experiment: false) + team = project_team(project) reviewers, traintainers, maintainers = %i[reviewer traintainer maintainer].map do |role| spin_role_for_category(team, role, project, category) @@ -132,11 +151,11 @@ module Gitlab # https://gitlab.com/gitlab-org/gitlab/issues/26723 # Make traintainers have triple the chance to be picked as a reviewer - random = new_random(branch_name) + random = new_random(mr_source_branch) reviewer = spin_for_person(reviewers + traintainers + traintainers, random: random, timezone_experiment: timezone_experiment) maintainer = spin_for_person(maintainers, random: random, timezone_experiment: timezone_experiment) - Spin.new(category, reviewer, maintainer) + Spin.new(category, reviewer, maintainer, false, timezone_experiment) end end end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index f7da66e77cd..9b389907090 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -3,10 +3,11 @@ module Gitlab module Danger class Teammate - attr_reader :username, :name, :role, :projects, :available, :tz_offset_hours + attr_reader :options, :username, :name, :role, :projects, :available, :tz_offset_hours # The options data are produced by https://gitlab.com/gitlab-org/gitlab-roulette/-/blob/master/lib/team_member.rb def initialize(options = {}) + @options = options @username = options['username'] @name = options['name'] @markdown_name = options['markdown_name'] @@ -16,6 +17,16 @@ module Gitlab @tz_offset_hours = options['tz_offset_hours'] end + def to_h + options + end + + def ==(other) + return false unless other.respond_to?(:username) + + other.username == username + end + def in_project?(name) projects&.has_key?(name) end @@ -69,9 +80,9 @@ module Gitlab def offset_diff_compared_to_author(author) diff = floored_offset_hours - author.floored_offset_hours - return "same timezone as `@#{author.username}`" if diff.zero? + return "same timezone as `@#{author.username}`" if diff == 0 - ahead_or_behind = diff < 0 ? 'behind' : 'ahead' + ahead_or_behind = diff < 0 ? 'behind' : 'ahead of' pluralized_hours = pluralize(diff.abs, 'hour', 'hours') "#{pluralized_hours} #{ahead_or_behind} `@#{author.username}`" diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index d88ca6d7fe3..e7df9fd27f0 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -2,8 +2,6 @@ module Gitlab module Database - include Gitlab::Metrics::Methods - # Minimum PostgreSQL version requirement per documentation: # https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements MINIMUM_POSTGRES_VERSION = 11 @@ -24,6 +22,7 @@ module Gitlab # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html MAX_INT_VALUE = 2147483647 + MIN_INT_VALUE = -2147483648 # The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz: # https://www.postgresql.org/docs/9.1/static/datatype-datetime.html @@ -50,10 +49,6 @@ module Gitlab # It does not include the default public schema EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze - define_histogram :gitlab_database_transaction_seconds do - docstring "Time spent in database transactions, in seconds" - end - def self.config ActiveRecord::Base.configurations[Rails.env] end @@ -80,7 +75,7 @@ module Gitlab # @deprecated def self.postgresql? - adapter_name.casecmp('postgresql').zero? + adapter_name.casecmp('postgresql') == 0 end def self.read_only? @@ -363,8 +358,11 @@ module Gitlab # observe_transaction_duration is called from ActiveRecordBaseTransactionMetrics.transaction and used to # record transaction durations. def self.observe_transaction_duration(duration_seconds) - labels = Gitlab::Metrics::Transaction.current&.labels || {} - gitlab_database_transaction_seconds.observe(labels, duration_seconds) + if current_transaction = ::Gitlab::Metrics::Transaction.current + current_transaction.observe(:gitlab_database_transaction_seconds, duration_seconds) do + docstring "Time spent in database transactions, in seconds" + end + end rescue Prometheus::Client::LabelSetValidator::LabelSetError => err # Ensure that errors in recording these metrics don't affect the operation of the application Rails.logger.error("Unable to observe database transaction duration: #{err}") # rubocop:disable Gitlab/RailsLogger diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index ab069ce1da1..1762b81b7d8 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -16,6 +16,7 @@ # batch_count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) # batch_distinct_count(::Project, :creator_id) # batch_distinct_count(::Project.with_active_services.service_desk_enabled.where(time_period), start: ::User.minimum(:id), finish: ::User.maximum(:id)) +# batch_sum(User, :sign_in_count) module Gitlab module Database module BatchCount @@ -27,6 +28,10 @@ module Gitlab BatchCounter.new(relation, column: column).count(mode: :distinct, batch_size: batch_size, start: start, finish: finish) end + def batch_sum(relation, column, batch_size: nil, start: nil, finish: nil) + BatchCounter.new(relation, column: nil, operation: :sum, operation_args: [column]).count(batch_size: batch_size, start: start, finish: finish) + end + class << self include BatchCount end @@ -35,6 +40,7 @@ module Gitlab class BatchCounter FALLBACK = -1 MIN_REQUIRED_BATCH_SIZE = 1_250 + DEFAULT_SUM_BATCH_SIZE = 1_000 MAX_ALLOWED_LOOPS = 10_000 SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep ALLOWED_MODES = [:itself, :distinct].freeze @@ -43,13 +49,16 @@ module Gitlab DEFAULT_DISTINCT_BATCH_SIZE = 10_000 DEFAULT_BATCH_SIZE = 100_000 - def initialize(relation, column: nil) + def initialize(relation, column: nil, operation: :count, operation_args: nil) @relation = relation @column = column || relation.primary_key + @operation = operation + @operation_args = operation_args end def unwanted_configuration?(finish, batch_size, start) - batch_size <= MIN_REQUIRED_BATCH_SIZE || + (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) || + (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) || (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || start > finish end @@ -60,7 +69,7 @@ module Gitlab check_mode!(mode) # non-distinct have better performance - batch_size ||= mode == :distinct ? DEFAULT_DISTINCT_BATCH_SIZE : DEFAULT_BATCH_SIZE + batch_size ||= batch_size_for_mode_and_operation(mode, @operation) start = actual_start(start) finish = actual_finish(finish) @@ -91,11 +100,17 @@ module Gitlab def batch_fetch(start, finish, mode) # rubocop:disable GitlabSecurity/PublicSend - @relation.select(@column).public_send(mode).where(between_condition(start, finish)).count + @relation.select(@column).public_send(mode).where(between_condition(start, finish)).send(@operation, *@operation_args) end private + def batch_size_for_mode_and_operation(mode, operation) + return DEFAULT_SUM_BATCH_SIZE if operation == :sum + + mode == :distinct ? DEFAULT_DISTINCT_BATCH_SIZE : DEFAULT_BATCH_SIZE + end + def between_condition(start, finish) return @column.between(start..(finish - 1)) if @column.is_a?(Arel::Attributes::Attribute) diff --git a/lib/gitlab/database/connection_timer.rb b/lib/gitlab/database/connection_timer.rb index ef8d52ba71c..f9b893ffd0f 100644 --- a/lib/gitlab/database/connection_timer.rb +++ b/lib/gitlab/database/connection_timer.rb @@ -23,7 +23,7 @@ module Gitlab end def interval_with_randomization - interval + rand(RANDOMIZATION_INTERVAL) if interval.positive? + interval + rand(RANDOMIZATION_INTERVAL) if interval > 0 end def current_clock_value diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 006a24da8fe..a618a3017b2 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -815,7 +815,7 @@ module Gitlab BEFORE INSERT OR UPDATE ON #{table} FOR EACH ROW - EXECUTE PROCEDURE #{trigger}() + EXECUTE FUNCTION #{trigger}() EOF end @@ -1062,7 +1062,7 @@ into similar problems in the future (e.g. when new tables are created). AND pg_class.relname = '#{table}' SQL - connection.select_value(check_sql).positive? + connection.select_value(check_sql) > 0 end # Adds a check constraint to a table diff --git a/lib/gitlab/database/partitioning/partition_creator.rb b/lib/gitlab/database/partitioning/partition_creator.rb index 348dd1ba660..4c1b13fe3b5 100644 --- a/lib/gitlab/database/partitioning/partition_creator.rb +++ b/lib/gitlab/database/partitioning/partition_creator.rb @@ -24,7 +24,7 @@ module Gitlab end def create_partitions - return unless Feature.enabled?(:postgres_dynamic_partition_creation, default_enabled: true) + Gitlab::AppLogger.info("Checking state of dynamic postgres partitions") models.each do |model| # Double-checking before getting the lease: diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index b676767f41d..e6d8ec55319 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -16,7 +16,9 @@ module Gitlab BATCH_SIZE = 50_000 # Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a timestamp column. - # One partition is created per month between the given `min_date` and `max_date`. + # One partition is created per month between the given `min_date` and `max_date`. Also installs a trigger on + # the original table to copy writes into the partitioned table. To copy over historic data from before creation + # of the partitioned table, use the `enqueue_partitioning_data_migration` helper in a post-deploy migration. # # A copy of the original table is required as PG currently does not support partitioning existing tables. # @@ -56,10 +58,10 @@ module Gitlab create_range_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key) create_daterange_partitions(partitioned_table_name, partition_column.name, min_date, max_date) create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key) - enqueue_background_migration(table_name, partitioned_table_name, primary_key) end - # Clean up a partitioned copy of an existing table. This deletes the partitioned table and all partitions. + # Clean up a partitioned copy of an existing table. First, deletes the database function and trigger that were + # used to copy writes to the partitioned table, then removes the partitioned table (also removing partitions). # # Example: # @@ -69,8 +71,6 @@ module Gitlab assert_table_is_allowed(table_name) assert_not_in_transaction_block(scope: ERROR_SCOPE) - cleanup_migration_jobs(table_name) - with_lock_retries do trigger_name = make_sync_trigger_name(table_name) drop_trigger(table_name, trigger_name) @@ -83,6 +83,38 @@ module Gitlab drop_table(partitioned_table_name) end + # Enqueue the background jobs that will backfill data in the partitioned table, by batch-copying records from + # original table. This helper should be called from a post-deploy migration. + # + # Example: + # + # enqueue_partitioning_data_migration :audit_events + # + def enqueue_partitioning_data_migration(table_name) + assert_table_is_allowed(table_name) + + assert_not_in_transaction_block(scope: ERROR_SCOPE) + + partitioned_table_name = make_partitioned_table_name(table_name) + primary_key = connection.primary_key(table_name) + enqueue_background_migration(table_name, partitioned_table_name, primary_key) + end + + # Cleanup a previously enqueued background migration to copy data into a partitioned table. This will not + # prevent the enqueued jobs from executing, but instead cleans up information in the database used to track the + # state of the background migration. It should be safe to also remove the partitioned table even if the + # background jobs are still in-progress, as the absence of the table will cause them to safely exit. + # + # Example: + # + # cleanup_partitioning_data_migration :audit_events + # + def cleanup_partitioning_data_migration(table_name) + assert_table_is_allowed(table_name) + + cleanup_migration_jobs(table_name) + end + def create_hash_partitions(table_name, number_of_partitions) transaction do (0..number_of_partitions - 1).each do |partition| diff --git a/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb b/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb new file mode 100644 index 00000000000..59bd24d3c37 --- /dev/null +++ b/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PostgresqlAdapter + module DumpSchemaVersionsMixin + extend ActiveSupport::Concern + + def dump_schema_information # :nodoc: + versions = schema_migration.all_versions + Gitlab::Database::SchemaVersionFiles.touch_all(versions) if versions.any? + + nil + end + end + end + end +end diff --git a/lib/gitlab/database/postgresql_adapter/schema_versions_copy_mixin.rb b/lib/gitlab/database/postgresql_adapter/schema_versions_copy_mixin.rb deleted file mode 100644 index d8f96643dcb..00000000000 --- a/lib/gitlab/database/postgresql_adapter/schema_versions_copy_mixin.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module PostgresqlAdapter - module SchemaVersionsCopyMixin - extend ActiveSupport::Concern - - def dump_schema_information # :nodoc: - versions = schema_migration.all_versions - copy_versions_sql(versions) if versions.any? - end - - private - - def copy_versions_sql(versions) - sm_table = quote_table_name(schema_migration.table_name) - - sql = +"COPY #{sm_table} (version) FROM STDIN;\n" - sql << versions.map { |v| Integer(v) }.sort.join("\n") - sql << "\n\\.\n" - - sql - end - end - end - end -end diff --git a/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb b/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb new file mode 100644 index 00000000000..cf8342941c4 --- /dev/null +++ b/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PostgresqlDatabaseTasks + module LoadSchemaVersionsMixin + extend ActiveSupport::Concern + + def structure_load(*args) + super(*args) + Gitlab::Database::SchemaVersionFiles.load_all + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 6b9af51a6ab..4fbbfdc4914 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -45,7 +45,7 @@ module Gitlab reverts_for_type('namespace') do |path_before_rename, current_path| matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) namespace = MigrationClasses::Namespace.joins(:route) - .find_by(matches_path)&.becomes(MigrationClasses::Namespace) + .find_by(matches_path)&.becomes(MigrationClasses::Namespace) # rubocop: disable Cop/AvoidBecomes if namespace perform_rename(namespace, current_path, path_before_rename) diff --git a/lib/gitlab/database/schema_cleaner.rb b/lib/gitlab/database/schema_cleaner.rb index ae9d77e635e..7c415287878 100644 --- a/lib/gitlab/database/schema_cleaner.rb +++ b/lib/gitlab/database/schema_cleaner.rb @@ -23,6 +23,11 @@ module Gitlab structure.gsub!(/\n{3,}/, "\n\n") io << structure + io << <<~MSG + -- schema_migrations.version information is no longer stored in this file, + -- but instead tracked in the db/schema_migrations directory + -- see https://gitlab.com/gitlab-org/gitlab/-/issues/218590 for details + MSG nil end diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index 34daafd06de..dda4d8eecdb 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -25,7 +25,7 @@ module Gitlab CREATE TRIGGER #{name} #{fires} ON #{table_name} FOR EACH ROW - EXECUTE PROCEDURE #{function_name}() + EXECUTE FUNCTION #{function_name}() SQL end diff --git a/lib/gitlab/database/schema_version_files.rb b/lib/gitlab/database/schema_version_files.rb new file mode 100644 index 00000000000..27a942404ef --- /dev/null +++ b/lib/gitlab/database/schema_version_files.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class SchemaVersionFiles + SCHEMA_DIRECTORY = 'db/schema_migrations' + MIGRATION_DIRECTORIES = %w[db/migrate db/post_migrate].freeze + MIGRATION_VERSION_GLOB = '20[0-9][0-9]*' + + def self.touch_all(versions_from_database) + versions_from_migration_files = find_versions_from_migration_files + + version_filepaths = find_version_filenames.map { |f| schema_directory.join(f) } + FileUtils.rm(version_filepaths) + + versions_to_create = versions_from_database & versions_from_migration_files + versions_to_create.each do |version| + version_filepath = schema_directory.join(version) + + File.open(version_filepath, 'w') do |file| + file << Digest::SHA256.hexdigest(version) + end + end + end + + def self.load_all + version_filenames = find_version_filenames + return if version_filenames.empty? + + values = version_filenames.map { |vf| "('#{connection.quote_string(vf)}')" } + connection.execute(<<~SQL) + INSERT INTO schema_migrations (version) + VALUES #{values.join(',')} + ON CONFLICT DO NOTHING + SQL + end + + def self.schema_directory + @schema_directory ||= Rails.root.join(SCHEMA_DIRECTORY) + end + + def self.migration_directories + @migration_directories ||= MIGRATION_DIRECTORIES.map { |dir| Rails.root.join(dir) } + end + + def self.find_version_filenames + Dir.glob(MIGRATION_VERSION_GLOB, base: schema_directory) + end + + def self.find_versions_from_migration_files + migration_directories.each_with_object([]) do |directory, migration_versions| + directory_migrations = Dir.glob(MIGRATION_VERSION_GLOB, base: directory) + directory_versions = directory_migrations.map! { |m| m.split('_').first } + + migration_versions.concat(directory_versions) + end + end + + def self.connection + ActiveRecord::Base.connection + end + end + end +end diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb new file mode 100644 index 00000000000..2633c29438a --- /dev/null +++ b/lib/gitlab/database/similarity_score.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class SimilarityScore + EMPTY_STRING = Arel.sql("''").freeze + EXPRESSION_ON_INVALID_INPUT = Arel::Nodes::NamedFunction.new('CAST', [Arel.sql('0').as('integer')]).freeze + DEFAULT_MULTIPLIER = 1 + + # This method returns an Arel expression that can be used in an ActiveRecord query to order the resultset by similarity. + # + # Note: Calculating similarity score for large volume of records is inefficient. use SimilarityScore only for smaller + # resultset which is already filtered by other conditions (< 10_000 records). + # + # ==== Parameters + # * +search+ - [String] the user provided search string + # * +rules+ - [{ column: COLUMN, multiplier: 1 }, { column: COLUMN_2, multiplier: 0.5 }] rules for the scoring. + # * +column+ - Arel column expression, example: Project.arel_table["name"] + # * +multiplier+ - Integer or Float to increase or decrease the score (optional, defaults to 1) + # + # ==== Use case + # + # We'd like to search for projects by path, name and description. We want to rank higher the path and name matches, since + # it's more likely that the user was remembering the path or the name of the project. + # + # Rules: + # [ + # { column: Project.arel_table['path'], multiplier: 1 }, + # { column: Project.arel_table['name'], multiplier: 1 }, + # { column: Project.arel_table['description'], multiplier: 0.5 } + # ] + # + # ==== Examples + # + # Similarity calculation based on one column: + # + # Gitlab::Database::SimilarityScore.build_expession(search: 'my input', rules: [{ column: Project.arel_table['name'] }]) + # + # Similarity calculation based on two column, where the second column has lower priority: + # + # Gitlab::Database::SimilarityScore.build_expession(search: 'my input', rules: [ + # { column: Project.arel_table['name'], multiplier: 1 }, + # { column: Project.arel_table['description'], multiplier: 0.5 } + # ]) + # + # Integration with an ActiveRecord query: + # + # table = Project.arel_table + # + # order_expression = Gitlab::Database::SimilarityScore.build_expession(search: 'input', rules: [ + # { column: table['name'], multiplier: 1 }, + # { column: table['description'], multiplier: 0.5 } + # ]) + # + # Project.where("name LIKE ?", '%' + 'input' + '%').order(order_expression.desc) + # + # The expression can be also used in SELECT: + # + # results = Project.select(order_expression.as('similarity')).where("name LIKE ?", '%' + 'input' + '%').order(similarity: :desc) + # puts results.map(&:similarity) + # + def self.build_expression(search:, rules:) + return EXPRESSION_ON_INVALID_INPUT if search.blank? || rules.empty? + + quoted_search = ActiveRecord::Base.connection.quote(search.to_s) + + first_expression, *expressions = rules.map do |rule| + rule_to_arel(quoted_search, rule) + end + + # (SIMILARITY ...) + (SIMILARITY ...) + expressions.inject(first_expression) do |expression1, expression2| + Arel::Nodes::Addition.new(expression1, expression2) + end + end + + # (SIMILARITY(COALESCE(column, ''), 'search_string') * CAST(multiplier AS numeric)) + def self.rule_to_arel(search, rule) + Arel::Nodes::Grouping.new( + Arel::Nodes::Multiplication.new( + similarity_function_call(search, column_expression(rule)), + multiplier_expression(rule) + ) + ) + end + + # COALESCE(column, '') + def self.column_expression(rule) + Arel::Nodes::NamedFunction.new('COALESCE', [rule.fetch(:column), EMPTY_STRING]) + end + + # SIMILARITY(COALESCE(column, ''), 'search_string') + def self.similarity_function_call(search, column) + Arel::Nodes::NamedFunction.new('SIMILARITY', [column, Arel.sql(search)]) + end + + # CAST(multiplier AS numeric) + def self.multiplier_expression(rule) + quoted_multiplier = ActiveRecord::Base.connection.quote(rule.fetch(:multiplier, DEFAULT_MULTIPLIER).to_s) + + Arel::Nodes::NamedFunction.new('CAST', [Arel.sql(quoted_multiplier).as('numeric')]) + end + + private_class_method :rule_to_arel + private_class_method :column_expression + private_class_method :similarity_function_call + private_class_method :multiplier_expression + end + end +end diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb index bebcba6f42e..a9c86e4e267 100644 --- a/lib/gitlab/database/with_lock_retries.rb +++ b/lib/gitlab/database/with_lock_retries.rb @@ -2,7 +2,14 @@ module Gitlab module Database + # This class provides a way to automatically execute code that relies on acquiring a database lock in a way + # designed to minimize impact on a busy production database. + # + # A default timing configuration is provided that makes repeated attempts to acquire the necessary lock, with + # varying lock_timeout settings, and also serves to limit the maximum number of attempts. class WithLockRetries + AttemptsExhaustedError = Class.new(StandardError) + NULL_LOGGER = Gitlab::JsonLogger.new('/dev/null') # Each element of the array represents a retry iteration. @@ -63,7 +70,17 @@ module Gitlab @log_params = { method: 'with_lock_retries', class: klass.to_s } end - def run(&block) + # Executes a block of code, retrying it whenever a database lock can't be acquired in time + # + # When a database lock can't be acquired, ActiveRecord throws ActiveRecord::LockWaitTimeout + # exception which we intercept to re-execute the block of code, until it finishes or we reach the + # max attempt limit. The default behavior when max attempts have been reached is to make a final attempt with the + # lock_timeout disabled, but this can be altered with the raise_on_exhaustion parameter. + # + # @see DEFAULT_TIMING_CONFIGURATION for the timings used when attempting a retry + # @param [Boolean] raise_on_exhaustion whether to raise `AttemptsExhaustedError` when exhausting max attempts + # @param [Proc] block of code that will be executed + def run(raise_on_exhaustion: false, &block) raise 'no block given' unless block_given? @block = block @@ -85,6 +102,9 @@ module Gitlab retry else reset_db_settings + + raise AttemptsExhaustedError, 'configured attempts to obtain locks are exhausted' if raise_on_exhaustion + run_block_without_lock_timeout end diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb index 59a7c4a6660..eb475307f27 100644 --- a/lib/gitlab/devise_failure.rb +++ b/lib/gitlab/devise_failure.rb @@ -7,8 +7,6 @@ module Gitlab # If the request format is not known, send a redirect instead of a 401 # response, since this is the outcome we're most likely to want def http_auth? - return super unless Feature.enabled?(:devise_redirect_unknown_formats, default_enabled: true) - request_format && super end diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb index 31eeadc45f7..e24150a2330 100644 --- a/lib/gitlab/diff/formatters/base_formatter.rb +++ b/lib/gitlab/diff/formatters/base_formatter.rb @@ -10,7 +10,6 @@ module Gitlab attr_reader :base_sha attr_reader :start_sha attr_reader :head_sha - attr_reader :position_type def initialize(attrs) if diff_file = attrs[:diff_file] diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index ccf09b37b9b..0c3b6b72313 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -3,7 +3,6 @@ module Gitlab module Diff class HighlightCache - include Gitlab::Metrics::Methods include Gitlab::Utils::StrongMemoize EXPIRATION = 1.week @@ -12,19 +11,6 @@ module Gitlab delegate :diffable, to: :@diff_collection delegate :diff_options, to: :@diff_collection - define_histogram :gitlab_redis_diff_caching_memory_usage_bytes do - docstring 'Redis diff caching memory usage by key' - buckets [100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000] - end - - define_counter :gitlab_redis_diff_caching_hit do - docstring 'Redis diff caching hits' - end - - define_counter :gitlab_redis_diff_caching_miss do - docstring 'Redis diff caching misses' - end - def initialize(diff_collection) @diff_collection = diff_collection end @@ -117,7 +103,10 @@ module Gitlab def record_memory_usage(memory_usage) if memory_usage - self.class.gitlab_redis_diff_caching_memory_usage_bytes.observe({}, memory_usage) + current_transaction&.observe(:gitlab_redis_diff_caching_memory_usage_bytes, memory_usage) do + docstring 'Redis diff caching memory usage by key' + buckets [100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000] + end end end @@ -163,34 +152,24 @@ module Gitlab end def compose_data(json_data) - if ::Feature.enabled?(:gzip_diff_cache, default_enabled: true) - # #compress returns ASCII-8BIT, so we need to force the encoding to - # UTF-8 before caching it in redis, else we risk encoding mismatch - # errors. - # - ActiveSupport::Gzip.compress(json_data).force_encoding("UTF-8") - else - json_data - end + # #compress returns ASCII-8BIT, so we need to force the encoding to + # UTF-8 before caching it in redis, else we risk encoding mismatch + # errors. + # + ActiveSupport::Gzip.compress(json_data).force_encoding("UTF-8") rescue Zlib::GzipFile::Error json_data end def extract_data(data) - # Since when we deploy this code, we'll be dealing with an already - # populated cache full of data that isn't gzipped, we want to also - # check to see if the data is gzipped before we attempt to #decompress - # it, thus we check the first 2 bytes for "\x1F\x8B" to confirm it is - # a gzipped string. While a non-gzipped string will raise a - # Zlib::GzipFile::Error, which we're rescuing, we don't want to count - # on rescue for control flow. This check can be removed in the release - # after this change is released. + # Since we could be dealing with an already populated cache full of data + # that isn't gzipped, we want to also check to see if the data is + # gzipped before we attempt to #decompress it, thus we check the first + # 2 bytes for "\x1F\x8B" to confirm it is a gzipped string. While a + # non-gzipped string will raise a Zlib::GzipFile::Error, which we're + # rescuing, we don't want to count on rescue for control flow. # - if ::Feature.enabled?(:gzip_diff_cache, default_enabled: true) && data[0..1] == "\x1F\x8B" - ActiveSupport::Gzip.decompress(data) - else - data - end + data[0..1] == "\x1F\x8B" ? ActiveSupport::Gzip.decompress(data) : data rescue Zlib::GzipFile::Error data end @@ -206,6 +185,10 @@ module Gitlab # @diff_collection.raw_diff_files end + + def current_transaction + ::Gitlab::Metrics::Transaction.current + end end end end diff --git a/lib/gitlab/diff/stats_cache.rb b/lib/gitlab/diff/stats_cache.rb index f38fb21d497..eb0ef4200dc 100644 --- a/lib/gitlab/diff/stats_cache.rb +++ b/lib/gitlab/diff/stats_cache.rb @@ -3,11 +3,11 @@ module Gitlab module Diff class StatsCache - include Gitlab::Metrics::Methods include Gitlab::Utils::StrongMemoize EXPIRATION = 1.week - VERSION = 1 + # The DiffStats#as_json representation is tied to the Gitaly protobuf version + VERSION = Gem.loaded_specs['gitaly'].version.to_s def initialize(cachable_key:) @cachable_key = cachable_key @@ -29,7 +29,8 @@ module Gitlab return if cache.exist?(key) return unless stats - cache.write(key, stats.as_json, expires_in: EXPIRATION) + cache.write(key, stats.map(&:to_h).as_json, expires_in: EXPIRATION) + clear_memoization(:cached_values) end def clear diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb index 2da1b8915e4..6aff8f909f3 100644 --- a/lib/gitlab/exception_log_formatter.rb +++ b/lib/gitlab/exception_log_formatter.rb @@ -23,7 +23,7 @@ module Gitlab end if exception.backtrace - payload['exception.backtrace'] = Gitlab::BacktraceCleaner.clean_backtrace(exception.backtrace) + payload['exception.backtrace'] = Rails.backtrace_cleaner.clean(exception.backtrace) end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 1d2c1c69423..b602393b59e 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -102,7 +102,7 @@ module Gitlab Gitlab::Redis::SharedState.with do |redis| ttl = redis.ttl(@redis_shared_state_key) - ttl if ttl.positive? + ttl if ttl > 0 end end diff --git a/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb b/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb index 8213c9bc042..52035220a71 100644 --- a/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb +++ b/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb @@ -39,7 +39,7 @@ module Gitlab end def first_attempt? - attempts.zero? + attempts == 0 end def sleep_sec diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index d3df9be0d63..9908369426a 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -53,9 +53,21 @@ module Gitlab }, new_create_project_ui: { tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi' + }, + terms_opt_in: { + tracking_category: 'Growth::Acquisition::Experiment::TermsOptIn' + }, + contact_sales_btn_in_app: { + tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp' + }, + customize_homepage: { + tracking_category: 'Growth::Expansion::Experiment::CustomizeHomepage' } }.freeze + GROUP_CONTROL = :control + GROUP_EXPERIMENTAL = :experimental + # Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent. # Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method # to controllers and views. It returns true when the experiment is enabled and the user is selected as part @@ -100,6 +112,12 @@ module Gitlab end end + def record_experiment_user(experiment_key) + return unless Experimentation.enabled?(experiment_key) && current_user + + ::Experiment.add_user(experiment_key, tracking_group(experiment_key), current_user) + end + private def dnt_enabled? @@ -126,7 +144,7 @@ module Gitlab { category: tracking_category(experiment_key), action: action, - property: tracking_group(experiment_key), + property: "#{tracking_group(experiment_key)}_group", label: experimentation_subject_id, value: value }.compact @@ -139,7 +157,7 @@ module Gitlab def tracking_group(experiment_key) return unless Experimentation.enabled?(experiment_key) - experiment_enabled?(experiment_key) ? 'experimental_group' : 'control_group' + experiment_enabled?(experiment_key) ? GROUP_EXPERIMENTAL : GROUP_CONTROL end def forced_enabled?(experiment_key) @@ -167,7 +185,7 @@ module Gitlab Experiment = Struct.new(:key, :environment, :tracking_category, keyword_init: true) do def enabled? - experiment_percentage.positive? + experiment_percentage > 0 end def enabled_for_environment? diff --git a/lib/gitlab/external_authorization/client.rb b/lib/gitlab/external_authorization/client.rb index 7985e6dcf7b..fc859304eab 100644 --- a/lib/gitlab/external_authorization/client.rb +++ b/lib/gitlab/external_authorization/client.rb @@ -17,23 +17,28 @@ module Gitlab end def request_access - response = Excon.post( + response = Gitlab::HTTP.post( service_url, post_params ) ::Gitlab::ExternalAuthorization::Response.new(response) - rescue Excon::Error => e + rescue *Gitlab::HTTP::HTTP_ERRORS => e raise ::Gitlab::ExternalAuthorization::RequestFailed.new(e) end private + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_system_hooks? + end + def post_params params = { headers: REQUEST_HEADERS, body: body.to_json, connect_timeout: timeout, read_timeout: timeout, - write_timeout: timeout } + write_timeout: timeout, + allow_local_requests: allow_local_requests? } if has_tls? params[:client_cert_data] = client_cert diff --git a/lib/gitlab/external_authorization/response.rb b/lib/gitlab/external_authorization/response.rb index 04f9688fad0..8656065303d 100644 --- a/lib/gitlab/external_authorization/response.rb +++ b/lib/gitlab/external_authorization/response.rb @@ -5,16 +5,16 @@ module Gitlab class Response include ::Gitlab::Utils::StrongMemoize - def initialize(excon_response) - @excon_response = excon_response + def initialize(response) + @response = response end def valid? - @excon_response && [200, 401, 403].include?(@excon_response.status) + @response && [200, 401, 403].include?(@response.code) end def successful? - valid? && @excon_response.status == 200 + valid? && @response.code == 200 end def reason @@ -28,7 +28,7 @@ module Gitlab end def parse_response! - Gitlab::Json.parse(@excon_response.body) + Gitlab::Json.parse(@response.body) rescue JSON::JSONError # The JSON response is optional, so don't fail when it's missing nil diff --git a/lib/gitlab/file_hook.rb b/lib/gitlab/file_hook.rb index f23ef2921d7..55eba2858fb 100644 --- a/lib/gitlab/file_hook.rb +++ b/lib/gitlab/file_hook.rb @@ -27,7 +27,7 @@ module Gitlab end exit_status = result.status&.exitstatus - [exit_status.zero?, result.stderr] + [exit_status == 0, result.stderr] rescue => e [false, e.message] end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 1b49d356d29..5d91eb605e8 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -5,7 +5,6 @@ module Gitlab class Blob include Gitlab::BlobHelper include Gitlab::EncodingHelper - include Gitlab::Metrics::Methods extend Gitlab::Git::WrapsGitalyErrors # This number is the maximum amount of data that we want to display to @@ -25,19 +24,24 @@ module Gitlab LFS_POINTER_MIN_SIZE = 120.bytes LFS_POINTER_MAX_SIZE = 200.bytes - attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary + attr_accessor :size, :mode, :id, :commit_id, :loaded_size, :binary + attr_writer :name, :path, :data - define_counter :gitlab_blob_truncated_true do - docstring 'blob.truncated? == true' + def self.gitlab_blob_truncated_true + @gitlab_blob_truncated_true ||= ::Gitlab::Metrics.counter(:gitlab_blob_truncated_true, 'blob.truncated? == true') end - define_counter :gitlab_blob_truncated_false do - docstring 'blob.truncated? == false' + def self.gitlab_blob_truncated_false + @gitlab_blob_truncated_false ||= ::Gitlab::Metrics.counter(:gitlab_blob_truncated_false, 'blob.truncated? == false') end - define_histogram :gitlab_blob_size do - docstring 'Gitlab::Git::Blob size' - buckets [1_000, 5_000, 10_000, 50_000, 100_000, 500_000, 1_000_000] + def self.gitlab_blob_size + @gitlab_blob_size ||= ::Gitlab::Metrics.histogram( + :gitlab_blob_size, + 'Gitlab::Git::Blob size', + {}, + [1_000, 5_000, 10_000, 50_000, 100_000, 500_000, 1_000_000] + ) end class << self diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 8db73ecc480..0bc7ecccf5e 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -261,7 +261,7 @@ module Gitlab end def has_zero_stats? - stats.total.zero? + stats.total == 0 rescue true end @@ -423,7 +423,7 @@ module Gitlab end def message_from_gitaly_body - return @raw_commit.subject.dup if @raw_commit.body_size.zero? + return @raw_commit.subject.dup if @raw_commit.body_size == 0 return @raw_commit.body.dup if full_body_fetched_from_gitaly? if @raw_commit.body_size > MAX_COMMIT_MESSAGE_DISPLAY_SIZE diff --git a/lib/gitlab/git/pre_receive_error.rb b/lib/gitlab/git/pre_receive_error.rb index ef9b1bf5224..7a6f27179f0 100644 --- a/lib/gitlab/git/pre_receive_error.rb +++ b/lib/gitlab/git/pre_receive_error.rb @@ -16,8 +16,16 @@ module Gitlab SAFE_MESSAGE_REGEX = /^(#{SAFE_MESSAGE_PREFIXES.join('|')})\s*(?<safe_message>.+)/.freeze - def initialize(message = '') - super(sanitize(message)) + attr_reader :raw_message + + def initialize(message = '', user_message = '') + @raw_message = message + + if user_message.present? + super(sanitize(user_message)) + else + super(sanitize(message)) + end end private diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index ea7a6e84195..596b4e9f692 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -44,7 +44,7 @@ module Gitlab # Relative path of repo attr_reader :relative_path - attr_reader :storage, :gl_repository, :relative_path, :gl_project_path + attr_reader :storage, :gl_repository, :gl_project_path # This remote name has to be stable for all types of repositories that # can join an object pool. If it's structure ever changes, a migration @@ -598,14 +598,15 @@ module Gitlab end end - def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run: false) args = { user: user, commit: commit, branch_name: branch_name, message: message, start_branch_name: start_branch_name, - start_repository: start_repository + start_repository: start_repository, + dry_run: dry_run } wrapped_gitaly_errors do @@ -613,14 +614,15 @@ module Gitlab end end - def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run: false) args = { user: user, commit: commit, branch_name: branch_name, message: message, start_branch_name: start_branch_name, - start_repository: start_repository + start_repository: start_repository, + dry_run: dry_run } wrapped_gitaly_errors do @@ -813,7 +815,7 @@ module Gitlab def fsck msg, status = gitaly_repository_client.fsck - raise GitError.new("Could not fsck repository: #{msg}") unless status.zero? + raise GitError.new("Could not fsck repository: #{msg}") unless status == 0 end def create_from_bundle(bundle_path) diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb index 5c73c0c66a9..dc869ff5279 100644 --- a/lib/gitlab/git/rugged_impl/blob.rb +++ b/lib/gitlab/git/rugged_impl/blob.rb @@ -48,7 +48,7 @@ module Gitlab name: blob_entry[:name], size: blob.size, # Rugged::Blob#content is expensive; don't call it if we don't have to. - data: limit.zero? ? '' : blob.content(limit), + data: limit == 0 ? '' : blob.content(limit), mode: blob_entry[:filemode].to_s(8), path: path, commit_id: sha, diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index 7e072c5db50..ed02f2e92ec 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -6,8 +6,8 @@ module Gitlab include Gitlab::EncodingHelper extend Gitlab::Git::WrapsGitalyErrors - attr_accessor :id, :root_id, :name, :path, :flat_path, :type, - :mode, :commit_id, :submodule_url + attr_accessor :id, :root_id, :type, :mode, :commit_id, :submodule_url + attr_writer :name, :path, :flat_path class << self # Get list of tree objects diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb index f6cac398548..a1f3d64ccde 100644 --- a/lib/gitlab/git/wiki_page.rb +++ b/lib/gitlab/git/wiki_page.rb @@ -3,7 +3,7 @@ module Gitlab module Git class WikiPage - attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical, :formatted_data + attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :historical, :formatted_data # This class abstracts away Gitlab::GitalyClient::WikiPage def initialize(gitaly_page, version) diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 37e3da984d6..f3b53a2ba0b 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -23,7 +23,6 @@ module Gitlab deploy_key_upload: 'This deploy key does not have write access to this project.', no_repo: 'A repository for this project does not exist yet.', project_not_found: 'The project you were looking for could not be found.', - namespace_not_found: 'The namespace you were looking for could not be found.', command_not_allowed: "The command you're trying to execute is not allowed.", upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', @@ -43,38 +42,42 @@ module Gitlab PUSH_COMMANDS = %w{git-receive-pack}.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :repository_path, :redirected_path, :auth_result_type, :changes, :logger + attr_reader :actor, :protocol, :authentication_abilities, + :namespace_path, :redirected_path, :auth_result_type, + :cmd, :changes + attr_accessor :container - alias_method :container, :project - - def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, repository_path: nil, redirected_path: nil, auth_result_type: nil) - @actor = actor - @project = project - @protocol = protocol + def initialize(actor, container, protocol, authentication_abilities:, namespace_path: nil, repository_path: nil, redirected_path: nil, auth_result_type: nil) + @actor = actor + @container = container + @protocol = protocol @authentication_abilities = Array(authentication_abilities) - @namespace_path = namespace_path || project&.namespace&.full_path - @repository_path = repository_path || project&.path + @namespace_path = namespace_path + @repository_path = repository_path @redirected_path = redirected_path @auth_result_type = auth_result_type end + def repository_path + @repository_path ||= project&.path + end + def check(cmd, changes) - @logger = Checks::TimedLogger.new(timeout: INTERNAL_TIMEOUT, header: LOG_HEADER) @changes = changes + @cmd = cmd check_protocol! check_valid_actor! check_active_user! - check_authentication_abilities!(cmd) - check_command_disabled!(cmd) - check_command_existence!(cmd) + check_authentication_abilities! + check_command_disabled! + check_command_existence! - custom_action = check_custom_action(cmd) + custom_action = check_custom_action return custom_action if custom_action - check_db_accessibility!(cmd) - check_namespace! - check_project!(cmd) + check_db_accessibility! + check_container! check_repository_existence! case cmd @@ -87,12 +90,27 @@ module Gitlab success_result end + def logger + @logger ||= Checks::TimedLogger.new(timeout: INTERNAL_TIMEOUT, header: LOG_HEADER) + end + def guest_can_download_code? - Guest.can?(:download_code, project) + Guest.can?(download_ability, container) end def user_can_download_code? - authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code) + authentication_abilities.include?(:download_code) && + user_access.can_do_action?(download_ability) + end + + # @return [Symbol] the name of a Declarative Policy ability to check + def download_ability + raise NotImplementedError + end + + # @return [Symbol] the name of a Declarative Policy ability to check + def push_ability + raise NotImplementedError end def build_can_download_code? @@ -111,13 +129,17 @@ module Gitlab private - def check_project!(_cmd) + def check_container! + check_project! if project? + end + + def check_project! check_project_accessibility! add_project_moved_message! end - def check_custom_action(cmd) - nil + def check_custom_action + # no-op: Overridden in EE end def check_for_console_messages @@ -152,12 +174,6 @@ module Gitlab end end - def check_namespace! - return if namespace_path.present? - - raise NotFoundError, ERROR_MESSAGES[:namespace_not_found] - end - def check_active_user! return unless user @@ -167,25 +183,29 @@ module Gitlab end end - def check_authentication_abilities!(cmd) + def check_authentication_abilities! case cmd when *DOWNLOAD_COMMANDS unless authentication_abilities.include?(:download_code) || authentication_abilities.include?(:build_download_code) - raise ForbiddenError, ERROR_MESSAGES[:auth_download] + raise ForbiddenError, error_message(:auth_download) end when *PUSH_COMMANDS unless authentication_abilities.include?(:push_code) - raise ForbiddenError, ERROR_MESSAGES[:auth_upload] + raise ForbiddenError, error_message(:auth_upload) end end end def check_project_accessibility! if project.blank? || !can_read_project? - raise NotFoundError, ERROR_MESSAGES[:project_not_found] + raise NotFoundError, not_found_message end end + def not_found_message + error_message(:project_not_found) + end + def add_project_moved_message! return if redirected_path.nil? @@ -194,34 +214,34 @@ module Gitlab project_moved.add_message end - def check_command_disabled!(cmd) - if upload_pack?(cmd) + def check_command_disabled! + if upload_pack? check_upload_pack_disabled! - elsif receive_pack?(cmd) + elsif receive_pack? check_receive_pack_disabled! end end def check_upload_pack_disabled! if http? && upload_pack_disabled_over_http? - raise ForbiddenError, ERROR_MESSAGES[:upload_pack_disabled_over_http] + raise ForbiddenError, error_message(:upload_pack_disabled_over_http) end end def check_receive_pack_disabled! if http? && receive_pack_disabled_over_http? - raise ForbiddenError, ERROR_MESSAGES[:receive_pack_disabled_over_http] + raise ForbiddenError, error_message(:receive_pack_disabled_over_http) end end - def check_command_existence!(cmd) + def check_command_existence! unless ALL_COMMANDS.include?(cmd) - raise ForbiddenError, ERROR_MESSAGES[:command_not_allowed] + raise ForbiddenError, error_message(:command_not_allowed) end end - def check_db_accessibility!(cmd) - return unless receive_pack?(cmd) + def check_db_accessibility! + return unless receive_pack? if Gitlab::Database.read_only? raise ForbiddenError, push_to_read_only_message @@ -229,9 +249,11 @@ module Gitlab end def check_repository_existence! - unless repository.exists? - raise NotFoundError, ERROR_MESSAGES[:no_repo] - end + raise NotFoundError, no_repo_message unless repository.exists? + end + + def no_repo_message + error_message(:no_repo) end def check_download_access! @@ -242,44 +264,62 @@ module Gitlab guest_can_download_code? unless passed - raise ForbiddenError, ERROR_MESSAGES[:download] + raise ForbiddenError, download_forbidden_message end end + def download_forbidden_message + error_message(:download) + end + + # We assume that all git-access classes are in project context by default. + # Override this method to be more specific. + def project? + true + end + + def project + container if container.is_a?(::Project) + end + def check_push_access! - if project.repository_read_only? - raise ForbiddenError, ERROR_MESSAGES[:read_only] + if container.repository_read_only? + raise ForbiddenError, error_message(:read_only) end if deploy_key? unless deploy_key.can_push_to?(project) - raise ForbiddenError, ERROR_MESSAGES[:deploy_key_upload] + raise ForbiddenError, error_message(:deploy_key_upload) end elsif user # User access is verified in check_change_access! else - raise ForbiddenError, ERROR_MESSAGES[:upload] + raise ForbiddenError, error_message(:upload) end check_change_access! end + def user_can_push? + user_access.can_do_action?(push_ability) + end + def check_change_access! # Deploy keys with write access can push anything return if deploy_key? if changes == ANY - can_push = user_access.can_do_action?(:push_code) || - project.any_branch_allows_collaboration?(user_access.user) + can_push = user_can_push? || + project&.any_branch_allows_collaboration?(user_access.user) unless can_push - raise ForbiddenError, ERROR_MESSAGES[:push_code] + raise ForbiddenError, error_message(:push_code) end else # If there are worktrees with a HEAD pointing to a non-existent object, # calls to `git rev-list --all` will fail in git 2.15+. This should also # clear stale lock files. - project.repository.clean_stale_repository_files + project.repository.clean_stale_repository_files if project.present? # Iterate over all changes to find if user allowed all of them to be applied changes_list.each.with_index do |change, index| @@ -293,16 +333,14 @@ module Gitlab end def check_single_change_access(change, skip_lfs_integrity_check: false) - change_access = Checks::ChangeAccess.new( + Checks::ChangeAccess.new( change, user_access: user_access, project: project, skip_lfs_integrity_check: skip_lfs_integrity_check, protocol: protocol, logger: logger - ) - - change_access.exec + ).validate! rescue Checks::TimedLogger::TimeoutError raise TimeoutError, logger.full_message end @@ -347,12 +385,12 @@ module Gitlab protocol == 'http' end - def upload_pack?(command) - command == 'git-upload-pack' + def upload_pack? + cmd == 'git-upload-pack' end - def receive_pack?(command) - command == 'git-receive-pack' + def receive_pack? + cmd == 'git-receive-pack' end def upload_pack_disabled_over_http? @@ -365,6 +403,16 @@ module Gitlab protected + def error_message(key) + self.class.ancestors.each do |cls| + return cls.const_get('ERROR_MESSAGES', false).fetch(key) + rescue NameError, KeyError + next + end + + raise ArgumentError, "No error message defined for #{key}" + end + def success_result ::Gitlab::GitAccessResult::Success.new(console_messages: check_for_console_messages) end @@ -374,9 +422,7 @@ module Gitlab end def user - return @user if defined?(@user) - - @user = + strong_memoize(:user) do case actor when User actor @@ -387,20 +433,21 @@ module Gitlab when :ci nil end + end end def user_access @user_access ||= if ci? CiAccess.new elsif user && request_from_ci_build? - BuildAccess.new(user, project: project) + BuildAccess.new(user, container: container) else - UserAccess.new(user, project: project) + UserAccess.new(user, container: container) end end def push_to_read_only_message - ERROR_MESSAGES[:cannot_push_to_read_only] + error_message(:cannot_push_to_read_only) end def repository diff --git a/lib/gitlab/git_access_design.rb b/lib/gitlab/git_access_design.rb index 36604bd0b3b..6bea9fe53b3 100644 --- a/lib/gitlab/git_access_design.rb +++ b/lib/gitlab/git_access_design.rb @@ -2,6 +2,8 @@ module Gitlab class GitAccessDesign < GitAccess + extend ::Gitlab::Utils::Override + def check(_cmd, _changes) check_protocol! check_can_create_design! @@ -9,6 +11,11 @@ module Gitlab success_result end + override :push_ability + def push_ability + :create_design + end + private def check_protocol! @@ -18,7 +25,7 @@ module Gitlab end def check_can_create_design! - unless user&.can?(:create_design, project) + unless user_can_push? raise ::Gitlab::GitAccess::ForbiddenError, "You are not allowed to manage designs of this project" end end diff --git a/lib/gitlab/git_access_project.rb b/lib/gitlab/git_access_project.rb index c79a61c263e..cdefcc84f7d 100644 --- a/lib/gitlab/git_access_project.rb +++ b/lib/gitlab/git_access_project.rb @@ -6,21 +6,41 @@ module Gitlab CreationError = Class.new(StandardError) + ERROR_MESSAGES = { + namespace_not_found: 'The namespace you were looking for could not be found.' + }.freeze + + override :download_ability + def download_ability + :download_code + end + + override :push_ability + def push_ability + :push_code + end + private - override :check_project! - def check_project!(cmd) - ensure_project_on_push!(cmd) + override :check_container! + def check_container! + check_namespace! + ensure_project_on_push! super end - def ensure_project_on_push!(cmd) - return if project || deploy_key? - return unless receive_pack?(cmd) && changes == ANY && authentication_abilities.include?(:push_code) + def check_namespace! + raise NotFoundError, ERROR_MESSAGES[:namespace_not_found] unless namespace_path.present? + end - namespace = Namespace.find_by_full_path(namespace_path) + def namespace + @namespace ||= Namespace.find_by_full_path(namespace_path) + end + def ensure_project_on_push! + return if project || deploy_key? + return unless receive_pack? && changes == ANY && authentication_abilities.include?(:push_code) return unless user&.can?(:create_projects, namespace) project_params = { @@ -35,8 +55,8 @@ module Gitlab raise CreationError, "Could not create project: #{project.errors.full_messages.join(', ')}" end - @project = project - user_access.project = @project + self.container = project + user_access.container = project Checks::ProjectCreated.new(repository, user, protocol).add_message end diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index 3de6c9ee30a..f2b4e930707 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -9,50 +9,68 @@ module Gitlab read_snippet: 'You are not allowed to read this snippet.', update_snippet: 'You are not allowed to update this snippet.', snippet_not_found: 'The snippet you were looking for could not be found.', - repository_not_found: 'The snippet repository you were looking for could not be found.' + no_repo: 'The snippet repository you were looking for could not be found.' }.freeze - attr_reader :snippet - - alias_method :container, :snippet + alias_method :snippet, :container def initialize(actor, snippet, protocol, **kwargs) - @snippet = snippet - - super(actor, snippet&.project, protocol, **kwargs) + super(actor, snippet, protocol, **kwargs) @auth_result_type = nil @authentication_abilities &= [:download_code, :push_code] end + override :check def check(cmd, changes) - # TODO: Investigate if expanding actor/authentication types are needed. - # https://gitlab.com/gitlab-org/gitlab/issues/202190 - if actor && !actor.is_a?(User) && !actor.instance_of?(Key) - raise ForbiddenError, ERROR_MESSAGES[:authentication_mechanism] - end - check_snippet_accessibility! super end + override :download_ability + def download_ability + :read_snippet + end + + override :push_ability + def push_ability + :update_snippet + end + private - override :check_namespace! - def check_namespace! - return unless snippet.is_a?(ProjectSnippet) + # TODO: Implement EE/Geo https://gitlab.com/gitlab-org/gitlab/issues/205629 + override :check_custom_action + def check_custom_action + # snippets never return custom actions, such as geo replication. + end - super + override :project? + def project? + project_snippet? + end + + override :project + def project + snippet&.project end - override :check_project! - def check_project!(cmd) - return unless snippet.is_a?(ProjectSnippet) + override :check_valid_actor! + def check_valid_actor! + # TODO: Investigate if expanding actor/authentication types are needed. + # https://gitlab.com/gitlab-org/gitlab/issues/202190 + if actor && !actor.is_a?(User) && !actor.instance_of?(Key) + raise ForbiddenError, ERROR_MESSAGES[:authentication_mechanism] + end super end + def project_snippet? + snippet.is_a?(ProjectSnippet) + end + override :check_push_access! def check_push_access! raise ForbiddenError, ERROR_MESSAGES[:update_snippet] unless user @@ -82,19 +100,9 @@ module Gitlab end end - override :guest_can_download_code? - def guest_can_download_code? - Guest.can?(:read_snippet, snippet) - end - - override :user_can_download_code? - def user_can_download_code? - authentication_abilities.include?(:download_code) && user_access.can_do_action?(:read_snippet) - end - override :check_change_access! def check_change_access! - unless user_access.can_do_action?(:update_snippet) + unless user_can_push? raise ForbiddenError, ERROR_MESSAGES[:update_snippet] end @@ -109,31 +117,19 @@ module Gitlab check_push_size! end - def check_single_change_access(change) + override :check_single_change_access + def check_single_change_access(change, _skip_lfs_integrity_check: false) Checks::SnippetCheck.new(change, logger: logger).validate! Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet.max_file_limit(user), logger: logger).validate! rescue Checks::TimedLogger::TimeoutError raise TimeoutError, logger.full_message end - override :check_repository_existence! - def check_repository_existence! - unless repository.exists? - raise NotFoundError, ERROR_MESSAGES[:repository_not_found] - end - end - override :user_access def user_access @user_access ||= UserAccessSnippet.new(user, snippet: snippet) end - # TODO: Implement EE/Geo https://gitlab.com/gitlab-org/gitlab/issues/205629 - override :check_custom_action - def check_custom_action(cmd) - nil - end - override :check_size_limit? def check_size_limit? return false if user&.migration_bot? diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index aad46937c32..a941282e713 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -2,41 +2,50 @@ module Gitlab class GitAccessWiki < GitAccess - prepend_if_ee('EE::Gitlab::GitAccessWiki') # rubocop: disable Cop/InjectEnterpriseEditionModule + extend ::Gitlab::Utils::Override ERROR_MESSAGES = { - read_only: "You can't push code to a read-only GitLab instance.", + download: 'You are not allowed to download files from this wiki.', + not_found: 'The wiki you were looking for could not be found.', + no_repo: 'A repository for this wiki does not exist yet.', + read_only: "You can't push code to a read-only GitLab instance.", write_to_wiki: "You are not allowed to write to this project's wiki." }.freeze - def guest_can_download_code? - Guest.can?(:download_wiki_code, project) + override :download_ability + def download_ability + :download_wiki_code end - def user_can_download_code? - authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code) + override :push_ability + def push_ability + :create_wiki end + override :check_change_access! def check_change_access! - unless user_access.can_do_action?(:create_wiki) - raise ForbiddenError, ERROR_MESSAGES[:write_to_wiki] - end - - if Gitlab::Database.read_only? - raise ForbiddenError, push_to_read_only_message - end + raise ForbiddenError, write_to_wiki_message unless user_can_push? true end def push_to_read_only_message - ERROR_MESSAGES[:read_only] + error_message(:read_only) end - private + def write_to_wiki_message + error_message(:write_to_wiki) + end + def not_found_message + error_message(:not_found) + end + + override :repository def repository - project.wiki.repository + container.wiki.repository end end end + +Gitlab::GitAccessWiki.prepend_if_ee('EE::Gitlab::GitAccessWiki') diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index b284aadc107..131c00db612 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -8,8 +8,6 @@ require 'grpc/health/v1/health_services_pb' module Gitlab module GitalyClient - include Gitlab::Metrics::Methods - class TooManyInvocationsError < StandardError attr_reader :call_site, :invocation_count, :max_call_stack @@ -191,11 +189,6 @@ module Gitlab Gitlab::SafeRequestStore[:gitaly_query_time] += duration end - def self.current_transaction_labels - Gitlab::Metrics::Transaction.current&.labels || {} - end - private_class_method :current_transaction_labels - # For some time related tasks we can't rely on `Time.now` since it will be # affected by Timecop in some tests, and the clock of some gitaly-related # components (grpc's c-core and gitaly server) use system time instead of @@ -483,7 +476,7 @@ module Gitlab return unless stack_counter max = max_call_count - return if max.zero? + return if max == 0 stack_counter.select { |_, v| v == max }.keys end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 87505418ae9..513063c60d2 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -179,7 +179,7 @@ module Gitlab ) if response.pre_receive_error.present? - raise Gitlab::Git::PreReceiveError.new("GL-HOOK-ERR: pre-receive hook failed.") + raise Gitlab::Git::PreReceiveError.new(response.pre_receive_error, "GL-HOOK-ERR: pre-receive hook failed.") end Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) @@ -187,24 +187,26 @@ module Gitlab raise Gitlab::Git::CommitError, e end - def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run: false) call_cherry_pick_or_revert(:cherry_pick, user: user, commit: commit, branch_name: branch_name, message: message, start_branch_name: start_branch_name, - start_repository: start_repository) + start_repository: start_repository, + dry_run: dry_run) end - def user_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + def user_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run: false) call_cherry_pick_or_revert(:revert, user: user, commit: commit, branch_name: branch_name, message: message, start_branch_name: start_branch_name, - start_repository: start_repository) + start_repository: start_repository, + dry_run: dry_run) end def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:, push_options: []) @@ -390,7 +392,7 @@ module Gitlab response end - def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run:) request_class = "Gitaly::User#{rpc.to_s.camelcase}Request".constantize request = request_class.new( @@ -400,7 +402,8 @@ module Gitlab branch_name: encode_binary(branch_name), message: encode_binary(message), start_branch_name: encode_binary(start_branch_name.to_s), - start_repository: start_repository.gitaly_repository + start_repository: start_repository.gitaly_repository, + dry_run: dry_run ) response = GitalyClient.call( diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 9da986ae921..34d1231b9a5 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -161,7 +161,7 @@ module Gitlab # The cache key may be empty to indicate a previously looked up user for # which we couldn't find an ID. - [exists, number.positive? ? number : nil] + [exists, number > 0 ? number : nil] end end end diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 2b482ee3d2d..2c0038b61e2 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -37,19 +37,19 @@ module Gitlab end def wiki? - self == WIKI + name == :wiki end def project? - self == PROJECT + name == :project end def snippet? - self == SNIPPET + name == :snippet end def design? - self == DESIGN + name == :design end def path_suffix diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index fbbfed7279d..dfba68ce899 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -43,10 +43,11 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend push_frontend_feature_flag(:snippets_vue, default_enabled: true) - push_frontend_feature_flag(:monaco_blobs, default_enabled: false) + push_frontend_feature_flag(:monaco_blobs, default_enabled: true) push_frontend_feature_flag(:monaco_ci, default_enabled: false) push_frontend_feature_flag(:snippets_edit_vue, default_enabled: false) push_frontend_feature_flag(:webperf_experiment, default_enabled: false) + push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/hashed_path.rb b/lib/gitlab/hashed_path.rb new file mode 100644 index 00000000000..2510c511e28 --- /dev/null +++ b/lib/gitlab/hashed_path.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# Class that returns the disk path for a model using hashed storage + +module Gitlab + class HashedPath + def initialize(*paths, root_hash:) + @paths = paths + @root_hash = root_hash + end + + def to_s + File.join(disk_hash[0..1], disk_hash[2..3], disk_hash, @paths.map(&:to_s)) + end + + alias_method :to_str, :to_s + + private + + def disk_hash + @disk_hash ||= Digest::SHA2.hexdigest(@root_hash.to_s) + end + end +end diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index 6a8e16f5a85..b72d08549fe 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -101,7 +101,7 @@ module Gitlab def any_non_empty_queue?(*workers) workers.any? do |worker| - !Sidekiq::Queue.new(worker.queue).size.zero? + Sidekiq::Queue.new(worker.queue).size != 0 # rubocop:disable Style/ZeroLengthPredicate end end diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 911b71c3734..559e1828a70 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -9,32 +9,51 @@ module Gitlab BlockedUrlError = Class.new(StandardError) RedirectionTooDeep = Class.new(StandardError) - HTTP_ERRORS = [ + HTTP_TIMEOUT_ERRORS = [ + Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout + ].freeze + HTTP_ERRORS = HTTP_TIMEOUT_ERRORS + [ SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, - Net::OpenTimeout, Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError, - Gitlab::HTTP::RedirectionTooDeep + Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep ].freeze + DEFAULT_TIMEOUT_OPTIONS = { + open_timeout: 10, + read_timeout: 20, + write_timeout: 30 + }.freeze + include HTTParty # rubocop:disable Gitlab/HTTParty + class << self + alias_method :httparty_perform_request, :perform_request + end + connection_adapter HTTPConnectionAdapter def self.perform_request(http_method, path, options, &block) - super + log_info = options.delete(:extra_log_info) + options_with_timeouts = + if !options.has_key?(:timeout) && Feature.enabled?(:http_default_timeouts) + options.with_defaults(DEFAULT_TIMEOUT_OPTIONS) + else + options + end + + httparty_perform_request(http_method, path, options_with_timeouts, &block) rescue HTTParty::RedirectionTooDeep raise RedirectionTooDeep - end - - def self.try_get(path, options = {}, &block) - log_info = options.delete(:extra_log_info) - self.get(path, options, &block) - rescue *HTTP_ERRORS => e extra_info = log_info || {} extra_info = log_info.call(e, path, options) if log_info.respond_to?(:call) - Gitlab::ErrorTracking.log_exception(e, extra_info) + raise e + end + + def self.try_get(path, options = {}, &block) + self.get(path, options, &block) + rescue *HTTP_ERRORS nil end end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 18f4cb559c5..3b19ae3d7ff 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -4,6 +4,20 @@ module Gitlab module I18n extend self + # Languages with less then 2% of available translations will not + # be available in the UI. + # https://gitlab.com/gitlab-org/gitlab/-/issues/221012 + NOT_AVAILABLE_IN_UI = %w[ + fil_PH + pl_PL + nl_NL + id_ID + cs_CZ + bg + eo + gl_ES + ].freeze + AVAILABLE_LANGUAGES = { 'bg' => 'Bulgarian - български', 'cs_CZ' => 'Czech - čeština', @@ -29,6 +43,10 @@ module Gitlab 'zh_TW' => 'Chinese, Traditional (Taiwan) - 繁體中文 (台灣)' }.freeze + def selectable_locales + AVAILABLE_LANGUAGES.reject { |key, _value| NOT_AVAILABLE_IN_UI.include? key } + end + def available_locales AVAILABLE_LANGUAGES.keys end diff --git a/lib/gitlab/i18n/html_todo.yml b/lib/gitlab/i18n/html_todo.yml new file mode 100644 index 00000000000..bfd96ba8579 --- /dev/null +++ b/lib/gitlab/i18n/html_todo.yml @@ -0,0 +1,315 @@ +# +# PLEASE DO NOT ADD NEW STRINGS TO THIS FILE. +# +# See https://docs.gitlab.com/ee/development/i18n/externalization.html#html +# for information on how to handle HTML in translations. + +# +# This file contains strings that need to be fixed to use the +# updated HTML guidelines. Any strings in this file will no +# longer be able to be translated until they have been updated. +# +# This file (and the functionality around it) will be removed +# once https://gitlab.com/gitlab-org/gitlab/-/issues/217933 is complete. +# +# See https://gitlab.com/gitlab-org/gitlab/-/issues/19485 for more details +# why this change has been made. +# + +" or <!merge request id>": + translations: + - " ወይም <!merge request id>" + - " ou <!merge request id>" + - " または <!merge request id>" + - "或 <!合併請求 id>" + - " или <!merge request id>" + - "或<!merge request id>" + - " або <!merge request id>" + - " oder <!merge request id>" + - " o <!merge request id>" + - " 또는 <!merge request id>" + - " o <!merge request id>" + - " veya <!merge request id>" + - " neu <!merge request id>" + - " neu <#issue id>" +" or <#issue id>": + translations: + - "或 <#issue id>" + - " ወይም ‹#issue id›" + - " ou <identificación #issue>" + - " ou <#issue id>" + - " または <#課題 ID>" + - " o <#issue id>" + - "或 <#議題 id>" + - " ou <#issue id>" + - " или <#issue id>" + - "或 <#issue id>" + - " або <#issue id>" + - " oder <#issue id>" + - " o <#issue id>" + - " 또는 <#issue id>" + - " ou <#issue id>" + - " o <#issue id>" + - " veya <#issue id>" + - " neu <#issue id>" +" or <&epic id>": + translations: + - " ወይም <&epic id>" + - " または <&エピックID>" + - " 或 <#史詩 id>" + - " или <&epic id>" + - " 或<#epic id>" + - " або <&epic id>" + - " oder <&epic id>" + - " o <&epic id>" + - " veya <&epic id>" + - " neu <#epic id>" + - " 또는 <&epic id>" +"< 1 hour": + translations: + - "1 時間未満" + - "< 1 小時" + - "< 1 часа" + - "< 1小时" + - "< 1 години" + - "< 1 hora" + - "< 1 saat" + - "< 1 Stunde" + - "< 1시간" + +# +# Strings below are fixed in the source code but the translations are still present in CrowdIn so the +# locale files will fail the linter. They can be deleted after next CrowdIn sync, likely in: +# https://gitlab.com/gitlab-org/gitlab/-/issues/226008 +# + +"This commit was signed with an <strong>unverified</strong> signature.": + plural_id: + translations: + - "このコミットは<strong>検証されていない</strong> 署名でサインされています。" + - "Этот коммит был подписан <strong>непроверенной</strong> подписью." + - "此提交使用 <strong>未经验证的</strong> 签名进行签名。" + - "Цей коміт підписано <strong>неперевіреним</strong> підписом." + - "Esta commit fue firmado con una firma <strong>no verificada</strong>." +"This commit was signed with a <strong>verified</strong> signature and the committer email is verified to belong to the same user.": + plural_id: + translations: + - "このコミットは <strong>検証済み</strong> の署名でサインされており、このコミッターのメールは同じユーザーのものであることが検証されています。" + - "Это коммит был подписан <strong>верифицированной</strong> подписью и коммитер подтвердил, что адрес почты принадлежит ему." + - "此提交使用 <strong>已验证</strong> 的签名进行签名，并且已验证提交者的电子邮件属于同一用户。" + - "Цей коміт підписано <strong>перевіреним</strong> підписом і адреса електронної пошти комітера гарантовано належить тому самому користувачу." + - "Este commit fue firmado con una firma verificada, y <strong>se ha verificado</strong> que la dirección de correo electrónico del committer y la firma pertenecen al mismo usuario." +"Branch <strong>%{branch_name}</strong> was created. To set up auto deploy, choose a GitLab CI Yaml template and commit your changes. %{link_to_autodeploy_doc}": + plural_id: + translations: + - "分支 <strong>%{branch_name}</strong> 已創建。如需設置自動部署， 請選擇合適的 GitLab CI Yaml 模板併提交更改。%{link_to_autodeploy_doc}" + - "O branch <strong>%{branch_name}</strong> foi criado. Para configurar o deploy automático, selecione um modelo de Yaml do GitLab CI e commit suas mudanças. %{link_to_autodeploy_doc}" + - "<strong>%{branch_name}</strong> ブランチが作成されました。自動デプロイを設定するには、GitLab CI Yaml テンプレートを選択して、変更をコミットしてください。 %{link_to_autodeploy_doc}" + - "La branch <strong>%{branch_name}</strong> è stata creata. Per impostare un rilascio automatico scegli un template CI di Gitlab e committa le tue modifiche %{link_to_autodeploy_doc}" + - "O ramo <strong>%{branch_name}</strong> foi criado. Para configurar a implantação automática, seleciona um modelo de Yaml do GitLab CI e envia as tuas alterações. %{link_to_autodeploy_doc}" + - "Ветка <strong>%{branch_name}</strong> создана. Для настройки автоматического развертывания выберите YAML-шаблон для GitLab CI и зафиксируйте свои изменения. %{link_to_autodeploy_doc}" + - "已创建分支 <strong>%{branch_name}</strong> 。如需设置自动部署， 请选择合适的 GitLab CI Yaml 模板并提交更改。%{link_to_autodeploy_doc}" + - "Гілка <strong>%{branch_name}</strong> створена. Для настройки автоматичного розгортання виберіть GitLab CI Yaml-шаблон і закомітьте зміни. %{link_to_autodeploy_doc}" + - "Клонът <strong>%{branch_name}</strong> беше създаден. За да настроите автоматичното внедряване, изберете Yaml шаблон за GitLab CI и подайте промените си. %{link_to_autodeploy_doc}" + - "Branch <strong>%{branch_name}</strong> wurde erstellt. Um die automatische Bereitstellung einzurichten, wähle eine GitLab CI Yaml Vorlage und committe deine Änderungen. %{link_to_autodeploy_doc}" + - "<strong>%{branch_name}</strong> 브랜치가 생성되었습니다. 자동 배포를 설정하려면 GitLab CI Yaml 템플릿을 선택하고 변경 사항을 적용하십시오. %{link_to_autodeploy_doc}" + - "La branĉo <strong>%{branch_name}</strong> estis kreita. Por agordi aŭtomatan disponigadon, bonvolu elekti Yaml-ŝablonon por GitLab CI kaj enmeti viajn ŝanĝojn. %{link_to_autodeploy_doc}" + - "La branche <strong>%{branch_name}</strong> a été créée. Pour mettre en place le déploiement automatisé, sélectionnez un modèle de fichier YAML pour l’intégration continue (CI) de GitLab, et validez les modifications. %{link_to_autodeploy_doc}" + - "La rama <strong>%{branch_name}</strong> fue creada. Para configurar el auto despliegue, escoge una plantilla Yaml para GitLab CI y envía tus cambios. %{link_to_autodeploy_doc}" +"GitLabPages|GitLab Pages are disabled for this project. You can enable them on your project's %{strong_start}Settings > General > Visibility%{strong_end} page.": + plural_id: + translations: + - "GitLab Pagesはこのプロジェクトでは無効になっています。 プロジェクトの%{strong_start} 設定> 全般> 可視性%{strong_end}ページで有効にできます。" + - "GitLab Pages отключены для этого проекта. Вы можете включить в поле %{strong_start}Настройки > Общие > Видимость%{strong_end} вашего проекта." + - "此项目禁用GitLab Pages。您可以在您的项目的%{strong_start}设置 > 常规 > 可见性%{strong_end} 页面启用。" + - "GitLab Pages вимкнено для цього проєкту. Ви можете їх увімкнути перейшовши на сторінку проєкту %{strong_start}Налаштування > Загальні > Видимість%{strong_end}." + - "Las páginas de GitLab están deshabilitadas para este proyecto. Puede habilitarlas en los ajustes %{strong_start} de su proyecto > General > Visibilidad%{strong_end}." +"You can invite a new member to <strong>%{project_name}</strong> or invite another group.": + plural_id: + translations: + - "新しいメンバーを<strong>%{project_name} </strong>に招待するか、別のグループを招待することができます。" + - "Podes convidar um novo para <strong>%{project_name}</strong> ou convidar outro grupo." + - "邀请新成员或另一个群组加入<strong>%{project_name}</strong>。" + - "Puede invitar a un nuevo miembro a <strong>%{project_name}</strong> o invitar a otro grupo." + - "<strong>%{project_name}</strong> projesine yeni bir üye davet edebilir veya başka bir grubu davet edebilirsiniz." + - "Вы можете пригласить нового участника в <strong>%{project_name}</strong> или пригласить другую группу." + - "Ви можете запросити нового учасника до <strong>%{project_name}</strong> або запросити іншу групу." +"You can invite a new member to <strong>%{project_name}</strong>.": + plural_id: + translations: + - "新しいメンバーを<strong>%{project_name} </strong>に招待できます。" + - "Podes convidar um novo membro para <strong>%{project_name}</strong>." + - "邀请新成员加入<strong>%{project_name}</strong>。" + - "Puedes invitar a un nuevo miembro a <strong>%{project_name}</strong>." + - "<strong>%{project_name}</strong> projesine yeni bir üye davet edebilirsiniz." + - "Вы можете пригласить нового участника в <strong>%{project_name}</strong>." + - "Ви можете запросити нового учасника до <strong>%{project_name}</strong>." +"You can invite another group to <strong>%{project_name}</strong>.": + plural_id: + translations: + - "他のグループを<strong>%{project_name} </strong>に招待できます。" + - "Podes convidar outro grupo para <strong>%{project_name}</strong>." + - "您可以邀请另一个群组加入<strong>%{project_name}</strong>。" + - "Ви можете запросити нову групу до <strong>%{project_name}</strong>." + - "Puedes invitar a otro grupo a <strong>%{project_name}</strong>." +"Example: <code>192.168.0.0/24</code>. %{read_more_link}.": + plural_id: + translations: +"Note that PostgreSQL %{pg_version_upcoming} will become the minimum required version in GitLab %{gl_version_upcoming} (%{gl_version_upcoming_date}). Please consider upgrading your environment to a supported PostgreSQL version soon, see <a href=\\\"%{pg_version_upcoming_url}\\\">the related epic</a> for details.": + plural_id: + translations: +"Authorize <strong>%{user}</strong> to use your account?": + plural_id: + translations: +"DeployFreeze|Specify times when deployments are not allowed for an environment. The <code>gitlab-ci.yml</code> file must be updated to make deployment jobs aware of the %{freeze_period_link_start}freeze period%{freeze_period_link_end}.": + plural_id: + translations: +"<project name>": + translations: + - "<название проекта>" + - "<project name>" + - "<proje adı>" + - "<naziv projekta>" + - "<ім’я проєкту>" + - "<프로젝트 이름>" +"<strong>Deletes</strong> source branch": + plural_id: + translations: + - "<strong>刪除</strong>來源分支" + - "<strong>Apagar</strong> branch de origem" + - "ソースブランチを<strong>削除</strong>" + - "<strong>刪除</strong>來源分支" + - "<strong>Apagar</strong> o ramo de origem" + - "<strong>Удаляет</strong> исходную ветку" + - "<strong>删除</strong>源分支" + - "<strong>Видаляє</strong> гілку-джерело" + - "<strong>Löscht</strong> den Quellbranch" + - "소스 브랜치 <strong>삭제</strong>" + - "<strong>Supprime</strong> la branche source" + - "<strong>elimina</strong> la rama origen" + - "Kaynak dalı <strong>siler</strong>" +"Badges|You are going to delete this badge. Deleted badges <strong>cannot</strong> be restored.": + plural_id: + translations: + - "Você está prestes a excluir este selo. Selos excluídos <strong>não podem</strong> ser restaurados." + - "このバッジを削除しようとしています。削除されたバッジは<strong>復元できません</strong>。" + - "Estás prestes a apagar este emblema. Emblemas apagados <strong>não podem</strong> ser restaurados." + - "Вы собираетесь удалить этот значок. Удаленные значки <strong>не могут</strong> быть восстановлены." + - "您即将删除此徽章。徽章被删除后 <strong>不能</strong> 恢复。" + - "Ви збираєтеся видалити цей значок. Вилучені значки <strong>не можуть</strong> бути відновлені." + - "Du bist gerade dabei dieses Badge zu entfernen. Entfernte Badges können <strong>nicht</strong> rückgängig gemacht werden." + - "이 배지를 삭제하려고합니다. 삭제 된 배지는 <strong>복원 할 수 없습니다</strong>." + - "Vous êtes sur le point de supprimer ce badge. Les badges supprimés <strong>ne peuvent pas</strong> être restaurés." + - "Va a eliminar esta insignia. Las insignias eliminadas <strong>no se pueden</strong> restaurar." + - "Bu rozeti sileceksiniz. Silinen rozetler geri <strong>yüklenemez</strong>." +"ClusterIntegration| This will permanently delete the following resources: <ul> <li>All installed applications and related resources</li> <li>The <code>gitlab-managed-apps</code> namespace</li> <li>Any project namespaces</li> <li><code>clusterroles</code></li> <li><code>clusterrolebindings</code></li> </ul>": + plural_id: + translations: + - "これにより、次のリソースは完全に削除されます <ul> <li>インストールされているすべてのアプリケーションと関連したリソース</li> <li> <code>gitlab-managed-apps</code> 名前空間</li> <li>任意のプロジェクト名前空間</li> <li><code>clusterroles</code></li> <li><code>clusterrolebindings</code></li> </ul>" + - "此操作将永久删除下列资源： <ul> <li>所有已安装的应用程序和相关资源</li> <li> <code>GitLab管理的应用</code> 命名空间</li> <li>任何项目命名空间</li> <li><code>clusterroles</code></li> <li><code>clusterrolebindings</code></li> </ul>" + - "Esto eliminará permanentemente los siguientes recursos: <ul> <li>Todas las aplicaciones instaladas y sus recursos relacionados</li> <li>El espacio de nombres <code>gitlab-managed-apps</code></li> <li>Cualquier espacio de nombres de proyecto</li> <li><code> clusterroles </code></li> <li><code>clusterrolebindings</code></li> </ul>" +"Configure a <code>.gitlab-webide.yml</code> file in the <code>.gitlab</code> directory to start using the Web Terminal. %{helpStart}Learn more.%{helpEnd}": + plural_id: + translations: + - "Configure um arquivo <code>.gitlab-webide.yml</code> no diretório <code>.gitlab</code> para começar a usar o Terminal Web. %{helpStart}Saiba mais.%{helpEnd}" + - "Webターミナルの使用を開始するには、 <code>.gitlab</code> ディレクトリの <code>.gitlab-webide.yml</code> ファイルを設定します。 詳細は%{helpStart}こちら%{helpEnd}です。" + - "Сконфигурируйте файл <code>.gitlab-webide.yml</code> в каталоге <code>.gitlab</code> чтобы начать использовать веб-терминал. %{helpStart}Узнайте больше.%{helpEnd}" + - "在 <code>.gitlab</code> 目录中配置 <code>.gitlab-webide.yml</code> 文件以开始使用Web终端。 %{helpStart}了解更多。%{helpEnd}" + - "Налаштуйте файл <code>.gitlab-webide.yml</code> у директорії <code>.gitlab</code>, щоб почати використовувати Веб-термінал. %{helpStart}Докладніше.%{helpEnd}" + - "웹 터미널 사용을 시작하도록 <code>.gitlab</code> 디렉토리에서 <code>.gitlab-webide.yml</code> 파일을 구성하십시오. %{helpStart}자세히 알아보십시오.%{helpEnd}" + - "Configure un archivo <code>.gitlab-webide.yml</code> en el directorio <code>.gitlab</code> para comenzar a utilizar el Terminal Web. %{helpStart}Aprende más.%{helpEnd}" +"Depends on <strong>%d closed</strong> merge request.": + plural_id: "Depends on <strong>%d closed</strong> merge requests." + translations: + - "В зависимости от <strong>%d закрытого</strong> запроса на слияние." + - "В зависимости от <strong>%d закрытых</strong> запросов на слияние." + - "В зависимости от <strong>%d закрытых</strong> запросов на слияние." + - "В зависимости от <strong>%d закрытых</strong> запросов на слияние." + - "依赖于<strong>%d个已关闭的</strong>合并请求" + - "Залежить від %d <strong>закритого</strong> запиту на злиття." + - "Залежить від %d <strong>закритих</strong> запитів на злиття." + - "Залежить від %d <strong>закритих</strong> запитів на злиття." + - "Залежить від %d <strong>закритих</strong> запитів на злиття." + - "<strong>%d kapanan</strong> birleştirme isteğine bağlıdır." + - "<strong>%d kapanan</strong> birleştirme isteğine bağlıdır." +"Go to <strong>Issues</strong> > <strong>Boards</strong> to access your personalized learning issue board.": + plural_id: + translations: + - "转至<strong>议题</strong> > <strong>看板</strong>访问您的个性化学习议题看板。" +"Labels|<span>Promote label</span> %{labelTitle} <span>to Group Label?</span>": + plural_id: + translations: + - "<span>要讓標籤</span> %{labelTitle} <span>提升到群組標籤嗎？</span>" + - "<span>Promover a etiqueta</span> %{labelTitle} <span>para etiqueta do Grupo?</span>" + - "%{labelTitle} <span>ラベルをグループラベルに昇格しますか？</span>" + - "<span>Повысить метку</span> %{labelTitle} <span>до групповой метки?</span>" + - "<span>将标记</span> %{labelTitle} <span>升级为群组标记？</span>" + - "<span>Перенести мітку</span> %{labelTitle} <span>на рівень групи?</span>" + - "<span>Label</span> %{labelTitle} <span>zu Gruppenlabel hochstufen?</span>" + - "<span>라벨</span> %{labelTitle} <span>(을)를 그룹 라벨로 승격하시겠습니까?</span>" + - "<span>Promouvoir l’étiquette</span> %{labelTitle} <span>en étiquette de groupe ?</span>" + - "<span>¿Promocionar la etiqueta</span> %{labelTitle} <span>a etiqueta de grupo?</span>" +"Lock this %{issuableDisplayName}? Only <strong>project members</strong> will be able to comment.": + plural_id: + translations: + - "Travar este %{issuableDisplayName}? Apenas <strong>membros do projeto</strong> poderão comentar." + - "%{issuableDisplayName} をロックしますか？<strong>プロジェクトメンバー</strong> のみコメントできます。" + - "锁定此%{issuableDisplayName}吗？锁定后将只有<strong>项目成员</strong>可以发表评论。" + - "Заблокувати цю %{issuableDisplayName}? Лише <strong>учасники проекту</strong> зможуть коментувати." + - "%{issuableDisplayName} sperren? Es werden nur noch <strong>Projektmitglieder</strong> kommentieren können." + - "Verrouiller ce·t·te %{issuableDisplayName} ? Seuls les <strong>membres du projet</strong> seront en mesure de commenter." + - "¿Bloquear este %{issuableDisplayName}? Sólo los <strong>miembros del proyecto</strong> podrán comentar." +"PrometheusService|<p class=\\\"text-tertiary\\\">No <a href=\\\"%{docsUrl}\\\">common metrics</a> were found</p>": + plural_id: + translations: + - "<p class=\\\"text-tertiary\\\">Nenhuma <a href=\\\"%{docsUrl}\\\">métrica comum</a> foi encontrada</p>" + - "<p class=\\\"text-tertiary\\\"><a href=\\\"%{docsUrl}\\\">共通メトリクス</a>は見つかりませんでした</p>" + - "<p class=\\\"text-tertiary\\\">Ни одной <a href=\\\"%{docsUrl}\\\">общей метрики</a> не найдено</p>" + - "<p class=\\\"text-tertiary\\\">无<a href=\\\"%{docsUrl}\\\">常用指标</a> </p>" + - "<p class=\\\"text-tertiary\\\">Ніяких <a href=\\\"%{docsUrl}\\\">загальних метрик</a> не знайдено</p>" + - "<p class=\\\"text-tertiary\\\">Es wurden keine <a href=\\\"%{docsUrl}\\\">allgemeinen Metriken</a> gefunden</p>" + - "<p class=\\\"text-tertiary\\\"><a href=\\\"%{docsUrl}\\\">공통 메트릭스</a>가 발견되지 않았습니다.</p>" + - "<p class=\\\"text-tertiary\\\">Aucune <a href=\\\"%{docsUrl}\\\">métrique commune</a> trouvée</p>" + - "<p class=\\\"text-tertiary\\\">No se han encontrado<a href=\\\"%{docsUrl}\\\">métricas comunes</a> </p>" +"This project does not have billing enabled. To create a cluster, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">enable billing <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> and try again.": + plural_id: + translations: + - "Este projeto não possui faturamento ativado. Para criar um cluster, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">ative o faturamento <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> e tente novamente." + - "このプロジェクトでは課金が有効になっていません。クラスターを作成するには、<a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\"> 課金を有効<i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> にして再度お試しください。" + - "此项目未启用账单。要创建群集，请 <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">启用账单 <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> 并重试。" + - "Для цього проекту вимкнено білінг. Щоб створити кластер, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">увімкніть білінг <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> і спробуйте знову." + - "Für dieses Projekt ist keine Abrechnung aktiviert. Um ein Cluster zu erstellen, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">aktiviere die Abrechnung<i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> und versuche es erneut." + - "Ce projet n’a pas de facturation activée. Afin de créer une grappe de serveurs, veuillez <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">activer la facturation<i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> et réessayer." + - "Este proyecto no tiene la facturación habilitada. Para crear un clúster, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">habilite la facturación <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> e inténtelo de nuevo." +"Unlock this %{issuableDisplayName}? <strong>Everyone</strong> will be able to comment.": + plural_id: + translations: + - "Desbloquear este %{issuableDisplayName}? <strong>Todos</strong> poderão comentar." + - "%{issuableDisplayName} のロックを解除しますか？ <strong>全員</strong>がコメントできるようになります。" + - "解锁此%{issuableDisplayName}吗？解锁后<strong>所有人</strong>都将可以发表评论。" + - "Розблокувати %{issuableDisplayName}? <strong>Будь-хто</strong> зможе залишати коментарі." + - "Dieses %{issuableDisplayName} entsperren? <strong>Jeder</strong> wird in der Lage sein zu kommentieren." + - "%{issuableDisplayName}(을)를 잠금해제 하시겠습니까? <strong>모두가</strong> 코멘트 할 수 있게 됩니다." + - "Déverrouiller %{issuableDisplayName} ? <strong>Tout le monde</strong> sera en mesure de commenter." + - "Desbloquear este %{issuableDisplayName}? <strong>Todos</strong> podrán comentar." +"confidentiality|You are going to turn off the confidentiality. This means <strong>everyone</strong> will be able to see and leave a comment on this issue.": + plural_id: + translations: + - "Você está prestes a desligar a confidencialidade. Isso significa que <strong>todos</strong> serão capazes de ver e deixar comentários nesse issue." + - "あなたは公開設定に変更しようとしています。これは<strong>すべての人</strong> が閲覧可能になり、課題に対してコメントを残すことができるようになることを意味します。" + - "即将关闭私密性。这将使得 <strong>所有用户</strong>都可以查看并且评论当前议题。" + - "Ви вимикаєте конфіденційність. Це означає, що <strong>будь-хто</strong> зможе бачити і залишати коментарі для цієї задачі." + - "Du willst die Vertraulichkeit deaktivieren. Das bedeutet, dass <strong>alle</strong> das Ticket betrachten und kommentieren können." + - "Vous êtes sur le point de désactiver la confidentialité. Cela signifie que <strong>tout le monde</strong> sera en mesure de voir et de laisser un commentaire sur ce ticket." + - "Va a desactivar la confidencialidad. Esto significa que <strong>todos</strong> podrán ver y dejar un comentario sobre este tema." +"confidentiality|You are going to turn on the confidentiality. This means that only team members with <strong>at least Reporter access</strong> are able to see and leave comments on the issue.": + plural_id: + translations: + - "Você está prestes a ligar a confidencialidade. Isso significa que apenas membros da equipe com <strong>ao menos acesso de Relator</strong> serão capazes de ver e deixar comentários nesse issue." + - "あなたは公開設定に変更しようとしています。これはチームに限定していた<strong>最小限の報告権限</strong>をなくし、課題に対してコメントを残すことができるようになることを意味します。" + - "即将设置私密性。这将使得 <strong>至少有Reporter以上权限</strong>的团队成员才能查看并且评论当前议题。" + - "Ви вмикаєте конфіденційність. Це означає що лише учасники команди <strong>рівня репортер або вище</strong> матимуть змогу бачити та залишати коментарі для цієї задачі." + - "Du willst die Vertraulichkeit aktivieren. Das bedeutet, dass nur Teammitglieder mit <strong>mindestens Reporter-Zugriff</strong> das Ticket betrachten und kommentieren können." + - "Vous êtes sur le point de d’activer la confidentialité. Cela signifie que seuls les membres de l’équipe avec <strong>au moins un accès en tant que rapporteur</strong> seront en mesure de voir et de laisser des commentaires sur le ticket." + - "Va a activar la confidencialidad. Esto significa que solo los miembros del equipo con como mínimo,<strong>acceso como Reporter</strong> podrán ver y dejar comentarios sobre la incidencia." + - "あなたは非公開設定をオンにしようとしています。これは、最低でも<strong>報告権限</strong>を持ったチームメンバーのみが課題を表示したりコメントを残したりすることができるようになるということです。" diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index c0687cd9b79..e56b88dfce0 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -5,13 +5,14 @@ module Gitlab class PoLinter include Gitlab::Utils::StrongMemoize - attr_reader :po_path, :translation_entries, :metadata_entry, :locale + attr_reader :po_path, :translation_entries, :metadata_entry, :locale, :html_todolist VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze - def initialize(po_path, locale = I18n.locale.to_s) + def initialize(po_path:, html_todolist:, locale: I18n.locale.to_s) @po_path = po_path @locale = locale + @html_todolist = html_todolist end def errors @@ -19,7 +20,7 @@ module Gitlab end def validate_po - if parse_error = parse_po + if (parse_error = parse_po) return 'PO-syntax errors' => [parse_error] end @@ -38,7 +39,11 @@ module Gitlab end @translation_entries = entries.map do |entry_data| - Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_forms) + Gitlab::I18n::TranslationEntry.new( + entry_data: entry_data, + nplurals: metadata_entry.expected_forms, + html_allowed: html_todolist.fetch(entry_data[:msgid], false) + ) end nil @@ -66,6 +71,7 @@ module Gitlab validate_newlines(errors, entry) validate_number_of_plurals(errors, entry) validate_unescaped_chars(errors, entry) + validate_html(errors, entry) validate_translation(errors, entry) errors @@ -85,6 +91,23 @@ module Gitlab end end + def validate_html(errors, entry) + common_message = 'contains < or >. Use variables to include HTML in the string, or the < and > codes ' \ + 'for the symbols. For more info see: https://docs.gitlab.com/ee/development/i18n/externalization.html#html' + + if entry.msgid_contains_potential_html? && !entry.msgid_html_allowed? + errors << common_message + end + + if entry.plural_id_contains_potential_html? && !entry.plural_id_html_allowed? + errors << 'plural id ' + common_message + end + + if entry.translations_contain_potential_html? && !entry.translations_html_allowed? + errors << 'translation ' + common_message + end + end + def validate_number_of_plurals(errors, entry) return unless metadata_entry&.expected_forms return unless entry.translated? diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb index 19c10b2e402..25a45332d27 100644 --- a/lib/gitlab/i18n/translation_entry.rb +++ b/lib/gitlab/i18n/translation_entry.rb @@ -4,12 +4,14 @@ module Gitlab module I18n class TranslationEntry PERCENT_REGEX = /(?:^|[^%])%(?!{\w*}|[a-z%])/.freeze + ANGLE_BRACKET_REGEX = /[<>]/.freeze - attr_reader :nplurals, :entry_data + attr_reader :nplurals, :entry_data, :html_allowed - def initialize(entry_data, nplurals) + def initialize(entry_data:, nplurals:, html_allowed:) @entry_data = entry_data @nplurals = nplurals + @html_allowed = html_allowed end def msgid @@ -83,8 +85,38 @@ module Gitlab string =~ PERCENT_REGEX end + def msgid_contains_potential_html? + contains_angle_brackets?(msgid) + end + + def plural_id_contains_potential_html? + contains_angle_brackets?(plural_id) + end + + def translations_contain_potential_html? + all_translations.any? { |translation| contains_angle_brackets?(translation) } + end + + def msgid_html_allowed? + html_allowed.present? + end + + def plural_id_html_allowed? + html_allowed.present? && html_allowed['plural_id'] == plural_id + end + + def translations_html_allowed? + msgid_html_allowed? && html_allowed['translations'].present? && all_translations.all? do |translation| + html_allowed['translations'].include?(translation) + end + end + private + def contains_angle_brackets?(string) + string =~ ANGLE_BRACKET_REGEX + end + def translation_entries @translation_entries ||= entry_data.fetch_values(*translation_keys) .reject(&:empty?) diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index bdecff0931c..2f8769e261d 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -47,8 +47,8 @@ module Gitlab def execute(cmd) output, status = Gitlab::Popen.popen(cmd) - @shared.error(Gitlab::ImportExport::Error.new(output.to_s)) unless status.zero? # rubocop:disable Gitlab/ModuleWithInstanceVariables - status.zero? + @shared.error(Gitlab::ImportExport::Error.new(output.to_s)) unless status == 0 # rubocop:disable Gitlab/ModuleWithInstanceVariables + status == 0 end def git_bin_path diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 3cb1eb72ceb..081745a49f4 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -28,7 +28,9 @@ module Gitlab copy_archive wait_for_archived_file do - validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: true) + # Disable archive validation by default + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/235949 + validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size) decompress_archive end rescue => e diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index aa961bd8d19..a240c367a42 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -205,6 +205,7 @@ excluded_attributes: - :state_id - :duplicated_to_id - :promoted_to_epic_id + - :blocking_issues_count merge_request: - :milestone_id - :sprint_id @@ -274,6 +275,7 @@ excluded_attributes: timelogs: - :issue_id - :merge_request_id + - :note_id notes: - :noteable_id - :review_id diff --git a/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb b/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb index cd947b15154..768c8bb4cbb 100644 --- a/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb +++ b/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb @@ -32,7 +32,7 @@ module Gitlab end def incident_created_at - Time.parse(incident_payload['created_at']) + Time.zone.parse(incident_payload['created_at']) rescue Time.current.utc # PagerDuty provides time in UTC end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 2889dbc68cc..d55906083ff 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -8,11 +8,11 @@ module Gitlab class << self def enabled? - config.enabled && config.address + config.enabled && config.address.present? end def supports_wildcard? - config.address && config.address.include?(WILDCARD_PLACEHOLDER) + config.address.present? && config.address.include?(WILDCARD_PLACEHOLDER) end def supports_issue_creation? diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 1df899747e0..0beab008f73 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -96,7 +96,7 @@ module Gitlab :gitlab_redis_client_requests_duration_seconds, 'Client side Redis request latency, per Redis server, excluding blocking commands', {}, - [0.005, 0.01, 0.1, 0.5] + [0.1, 0.5, 0.75, 1] ) @request_latency_histogram.observe({ storage: storage_key }, duration) diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb index 659fb1472d2..7be54a214dd 100644 --- a/lib/gitlab/issuables_count_for_state.rb +++ b/lib/gitlab/issuables_count_for_state.rb @@ -9,9 +9,16 @@ module Gitlab # The state values that can be safely casted to a Symbol. STATES = %w[opened closed merged all].freeze + attr_reader :project + + def self.declarative_policy_class + 'IssuablePolicy' + end + # finder - The finder class to use for retrieving the issuables. - def initialize(finder) + def initialize(finder, project = nil) @finder = finder + @project = project @cache = Gitlab::SafeRequestStore[CACHE_KEY] ||= initialize_cache end @@ -19,6 +26,11 @@ module Gitlab self[state || :opened] end + # Define method for each state + STATES.each do |state| + define_method(state) { self[state] } + end + # Returns the count for the given state. # # state - The name of the state as either a String or a Symbol. diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 21f837c58bb..d6681347f42 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -220,5 +220,33 @@ module Gitlab end end end + + class LimitedEncoder + LimitExceeded = Class.new(StandardError) + + # Generates JSON for an object or raise an error if the resulting json string is too big + # + # @param object [Hash, Array, Object] must be hash, array, or an object that responds to .to_h or .to_json + # @param limit [Integer] max size of the resulting json string + # @return [String] + # @raise [LimitExceeded] if the resulting json string is bigger than the specified limit + def self.encode(object, limit: 25.megabytes) + return ::Gitlab::Json.dump(object) unless Feature.enabled?(:json_limited_encoder) + + buffer = [] + buffer_size = 0 + + ::Yajl::Encoder.encode(object) do |data_chunk| + chunk_size = data_chunk.bytesize + + raise LimitExceeded if buffer_size + chunk_size > limit + + buffer << data_chunk + buffer_size += chunk_size + end + + buffer.join('') + end + end end end diff --git a/lib/gitlab/kubernetes/cilium_network_policy.rb b/lib/gitlab/kubernetes/cilium_network_policy.rb new file mode 100644 index 00000000000..55afd2b586e --- /dev/null +++ b/lib/gitlab/kubernetes/cilium_network_policy.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class CiliumNetworkPolicy + include NetworkPolicyCommon + extend ::Gitlab::Utils::Override + + API_VERSION = "cilium.io/v2" + KIND = 'CiliumNetworkPolicy' + + def initialize(name:, namespace:, selector:, ingress:, resource_version:, labels: nil, creation_timestamp: nil, egress: nil) + @name = name + @namespace = namespace + @labels = labels + @creation_timestamp = creation_timestamp + @selector = selector + @resource_version = resource_version + @ingress = ingress + @egress = egress + end + + def generate + ::Kubeclient::Resource.new.tap do |resource| + resource.kind = KIND + resource.apiVersion = API_VERSION + resource.metadata = metadata + resource.spec = spec + end + end + + def self.from_yaml(manifest) + return unless manifest + + policy = YAML.safe_load(manifest, symbolize_names: true) + return if !policy[:metadata] || !policy[:spec] + + metadata = policy[:metadata] + spec = policy[:spec] + self.new( + name: metadata[:name], + namespace: metadata[:namespace], + resource_version: metadata[:resourceVersion], + labels: metadata[:labels], + selector: spec[:endpointSelector], + ingress: spec[:ingress], + egress: spec[:egress] + ) + rescue Psych::SyntaxError, Psych::DisallowedClass + nil + end + + def self.from_resource(resource) + return unless resource + return if !resource[:metadata] || !resource[:spec] + + metadata = resource[:metadata] + spec = resource[:spec].to_h + self.new( + name: metadata[:name], + namespace: metadata[:namespace], + resource_version: metadata[:resourceVersion], + labels: metadata[:labels]&.to_h, + creation_timestamp: metadata[:creationTimestamp], + selector: spec[:endpointSelector], + ingress: spec[:ingress], + egress: spec[:egress] + ) + end + + private + + attr_reader :name, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress + + def selector + @selector ||= {} + end + + override :spec + def spec + { + endpointSelector: selector, + ingress: ingress, + egress: egress + } + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb index f27ad05599e..49d2969f7f3 100644 --- a/lib/gitlab/kubernetes/helm/base_command.rb +++ b/lib/gitlab/kubernetes/helm/base_command.rb @@ -6,21 +6,16 @@ module Gitlab class BaseCommand attr_reader :name, :files - def initialize(rbac:, name:, files:, local_tiller_enabled:) + def initialize(rbac:, name:, files:) @rbac = rbac @name = name @files = files - @local_tiller_enabled = local_tiller_enabled end def rbac? @rbac end - def local_tiller_enabled? - @local_tiller_enabled - end - def pod_resource pod_service_account_name = rbac? ? service_account_name : nil diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb index 24458e1b4b3..a9e93c0c90e 100644 --- a/lib/gitlab/kubernetes/helm/client_command.rb +++ b/lib/gitlab/kubernetes/helm/client_command.rb @@ -5,30 +5,11 @@ module Gitlab module Helm module ClientCommand def init_command - if local_tiller_enabled? - <<~HEREDOC.chomp + <<~SHELL.chomp export HELM_HOST="localhost:44134" tiller -listen ${HELM_HOST} -alsologtostderr & helm init --client-only - HEREDOC - else - # Here we are always upgrading to the latest version of Tiller when - # installing an app. We ensure the helm version stored in the - # database is correct by also updating this after transition to - # :installed,:updated in Clusters::Concerns::ApplicationStatus - 'helm init --upgrade' - end - end - - def wait_for_tiller_command - return if local_tiller_enabled? - - helm_check = ['helm', 'version', *optional_tls_flags].shelljoin - # This is necessary to give Tiller time to restart after upgrade. - # Ideally we'd be able to use --wait but cannot because of - # https://github.com/helm/helm/issues/4855 - - "for i in $(seq 1 30); do #{helm_check} && s=0 && break || s=$?; sleep 1s; echo \"Retrying ($i)...\"; done; (exit $s)" + SHELL end def repository_command @@ -37,12 +18,6 @@ module Gitlab private - def tls_flags_if_remote_tiller - return [] if local_tiller_enabled? - - optional_tls_flags - end - def repository_update_command 'helm repo update' end diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb index 3bb41d09994..f8b9601bc98 100644 --- a/lib/gitlab/kubernetes/helm/delete_command.rb +++ b/lib/gitlab/kubernetes/helm/delete_command.rb @@ -17,7 +17,6 @@ module Gitlab def generate_script super + [ init_command, - wait_for_tiller_command, predelete, delete_command, postdelete @@ -29,9 +28,7 @@ module Gitlab end def delete_command - command = ['helm', 'delete', '--purge', name] + tls_flags_if_remote_tiller - - command.shelljoin + ['helm', 'delete', '--purge', name].shelljoin end end end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index cf6d993cad4..d166842fce6 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -21,7 +21,6 @@ module Gitlab def generate_script super + [ init_command, - wait_for_tiller_command, repository_command, repository_update_command, preinstall, @@ -39,7 +38,6 @@ module Gitlab install_flag + rollback_support_flag + reset_values_flag + - tls_flags_if_remote_tiller + optional_version_flag + rbac_create_flag + namespace_flag + diff --git a/lib/gitlab/kubernetes/helm/patch_command.rb b/lib/gitlab/kubernetes/helm/patch_command.rb index 1a5fab116bd..a33dbdac134 100644 --- a/lib/gitlab/kubernetes/helm/patch_command.rb +++ b/lib/gitlab/kubernetes/helm/patch_command.rb @@ -26,7 +26,6 @@ module Gitlab def generate_script super + [ init_command, - wait_for_tiller_command, repository_command, repository_update_command, upgrade_command @@ -38,7 +37,6 @@ module Gitlab def upgrade_command command = ['helm', 'upgrade', name, chart] + reuse_values_flag + - tls_flags_if_remote_tiller + version_flag + namespace_flag + value_flag diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 2110d586d30..9e3cf58bb1e 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -21,7 +21,8 @@ module Gitlab istio: { group: 'apis/networking.istio.io', version: 'v1alpha3' }, knative: { group: 'apis/serving.knative.dev', version: 'v1alpha1' }, metrics: { group: 'apis/metrics.k8s.io', version: 'v1beta1' }, - networking: { group: 'apis/networking.k8s.io', version: 'v1' } + networking: { group: 'apis/networking.k8s.io', version: 'v1' }, + cilium_networking: { group: 'apis/cilium.io', version: 'v2' } }.freeze SUPPORTED_API_GROUPS.each do |name, params| @@ -95,6 +96,14 @@ module Gitlab :delete_network_policy, to: :networking_client + # CiliumNetworkPolicy methods delegate to the apis/cilium.io api + # group client + delegate :create_cilium_network_policy, + :get_cilium_network_policies, + :update_cilium_network_policy, + :delete_cilium_network_policy, + to: :cilium_networking_client + attr_reader :api_prefix, :kubeclient_options DEFAULT_KUBECLIENT_OPTIONS = { @@ -107,15 +116,15 @@ module Gitlab def self.graceful_request(cluster_id) { status: :connected, response: yield } rescue *Gitlab::Kubernetes::Errors::CONNECTION - { status: :unreachable } + { status: :unreachable, connection_error: :connection_error } rescue *Gitlab::Kubernetes::Errors::AUTHENTICATION - { status: :authentication_failure } + { status: :authentication_failure, connection_error: :authentication_error } rescue Kubeclient::HttpError => e - { status: kubeclient_error_status(e.message) } + { status: kubeclient_error_status(e.message), connection_error: :http_error } rescue => e Gitlab::ErrorTracking.track_exception(e, cluster_id: cluster_id) - { status: :unknown_failure } + { status: :unknown_failure, connection_error: :unknown_error } end # KubeClient uses the same error class diff --git a/lib/gitlab/kubernetes/network_policy.rb b/lib/gitlab/kubernetes/network_policy.rb index dc13a614551..28810dc4453 100644 --- a/lib/gitlab/kubernetes/network_policy.rb +++ b/lib/gitlab/kubernetes/network_policy.rb @@ -3,19 +3,27 @@ module Gitlab module Kubernetes class NetworkPolicy - DISABLED_BY_LABEL = :'network-policy.gitlab.com/disabled_by' + include NetworkPolicyCommon + extend ::Gitlab::Utils::Override - def initialize(name:, namespace:, pod_selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil) + def initialize(name:, namespace:, selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil) @name = name @namespace = namespace @labels = labels @creation_timestamp = creation_timestamp - @pod_selector = pod_selector + @selector = selector @policy_types = policy_types @ingress = ingress @egress = egress end + def generate + ::Kubeclient::Resource.new.tap do |resource| + resource.metadata = metadata + resource.spec = spec + end + end + def self.from_yaml(manifest) return unless manifest @@ -28,7 +36,7 @@ module Gitlab name: metadata[:name], namespace: metadata[:namespace], labels: metadata[:labels], - pod_selector: spec[:podSelector], + selector: spec[:podSelector], policy_types: spec[:policyTypes], ingress: spec[:ingress], egress: spec[:egress] @@ -48,81 +56,30 @@ module Gitlab namespace: metadata[:namespace], labels: metadata[:labels]&.to_h, creation_timestamp: metadata[:creationTimestamp], - pod_selector: spec[:podSelector], + selector: spec[:podSelector], policy_types: spec[:policyTypes], ingress: spec[:ingress], egress: spec[:egress] ) end - def generate - ::Kubeclient::Resource.new.tap do |resource| - resource.metadata = metadata - resource.spec = spec - end - end - - def as_json(opts = nil) - { - name: name, - namespace: namespace, - creation_timestamp: creation_timestamp, - manifest: manifest, - is_autodevops: autodevops?, - is_enabled: enabled? - } - end - - def autodevops? - return false unless labels - - !labels[:chart].nil? && labels[:chart].start_with?('auto-deploy-app-') - end - - # podSelector selects pods that should be targeted by this - # policy. We can narrow selection by requiring this policy to - # match our custom labels. Since DISABLED_BY label will not be - # on any pod a policy will be effectively disabled. - def enabled? - return true unless pod_selector&.key?(:matchLabels) - - !pod_selector[:matchLabels]&.key?(DISABLED_BY_LABEL) - end - - def enable - return if enabled? - - pod_selector[:matchLabels].delete(DISABLED_BY_LABEL) - end - - def disable - @pod_selector ||= {} - pod_selector[:matchLabels] ||= {} - pod_selector[:matchLabels].merge!(DISABLED_BY_LABEL => 'gitlab') - end - private - attr_reader :name, :namespace, :labels, :creation_timestamp, :pod_selector, :policy_types, :ingress, :egress + attr_reader :name, :namespace, :labels, :creation_timestamp, :policy_types, :ingress, :egress - def metadata - meta = { name: name, namespace: namespace } - meta[:labels] = labels if labels - meta + def selector + @selector ||= {} end + override :spec def spec { - podSelector: pod_selector, + podSelector: selector, policyTypes: policy_types, ingress: ingress, egress: egress } end - - def manifest - YAML.dump({ metadata: metadata, spec: spec }.deep_stringify_keys) - end end end end diff --git a/lib/gitlab/kubernetes/network_policy_common.rb b/lib/gitlab/kubernetes/network_policy_common.rb new file mode 100644 index 00000000000..3b6e46d21ef --- /dev/null +++ b/lib/gitlab/kubernetes/network_policy_common.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module NetworkPolicyCommon + DISABLED_BY_LABEL = :'network-policy.gitlab.com/disabled_by' + + def as_json(opts = nil) + { + name: name, + namespace: namespace, + creation_timestamp: creation_timestamp, + manifest: manifest, + is_autodevops: autodevops?, + is_enabled: enabled? + } + end + + def autodevops? + return false unless labels + + !labels[:chart].nil? && labels[:chart].start_with?('auto-deploy-app-') + end + + # selector selects pods that should be targeted by this + # policy. It can represent podSelector, nodeSelector or + # endpointSelector We can narrow selection by requiring + # this policy to match our custom labels. Since DISABLED_BY + # label will not be on any pod a policy will be effectively disabled. + def enabled? + return true unless selector&.key?(:matchLabels) + + !selector[:matchLabels]&.key?(DISABLED_BY_LABEL) + end + + def enable + return if enabled? + + selector[:matchLabels].delete(DISABLED_BY_LABEL) + end + + def disable + selector[:matchLabels] ||= {} + selector[:matchLabels].merge!(DISABLED_BY_LABEL => 'gitlab') + end + + private + + def metadata + meta = { name: name, namespace: namespace } + meta[:labels] = labels if labels + meta[:resourceVersion] = resource_version if defined?(resource_version) + meta + end + + def spec + raise NotImplementedError + end + + def manifest + YAML.dump({ metadata: metadata, spec: spec }.deep_stringify_keys) + end + end + end +end diff --git a/lib/gitlab/kubernetes/node.rb b/lib/gitlab/kubernetes/node.rb index bd765ef3852..d516bdde6f6 100644 --- a/lib/gitlab/kubernetes/node.rb +++ b/lib/gitlab/kubernetes/node.rb @@ -8,22 +8,29 @@ module Gitlab end def all - nodes.map do |node| - attributes = node(node) - attributes.merge(node_metrics(node)) - end + { + nodes: metadata.presence, + node_connection_error: nodes_from_cluster[:connection_error], + metrics_connection_error: nodes_metrics_from_cluster[:connection_error] + }.compact end private attr_reader :cluster + def metadata + nodes.map do |node| + base_data(node).merge(node_metrics(node)) + end + end + def nodes_from_cluster - graceful_request { cluster.kubeclient.get_nodes } + @nodes_from_cluster ||= graceful_request { cluster.kubeclient.get_nodes } end def nodes_metrics_from_cluster - graceful_request { cluster.kubeclient.metrics_client.get_nodes } + @nodes_metrics_from_cluster ||= graceful_request { cluster.kubeclient.metrics_client.get_nodes } end def nodes @@ -44,7 +51,7 @@ module Gitlab ::Gitlab::Kubernetes::KubeClient.graceful_request(cluster.id, &block) end - def node(node) + def base_data(node) { 'metadata' => { 'name' => node.metadata.name diff --git a/lib/gitlab/manifest_import/project_creator.rb b/lib/gitlab/manifest_import/project_creator.rb index 837d65e5f7c..6637cbb9cc8 100644 --- a/lib/gitlab/manifest_import/project_creator.rb +++ b/lib/gitlab/manifest_import/project_creator.rb @@ -18,6 +18,7 @@ module Gitlab params = { import_url: repository[:url], + import_source: repository[:url], import_type: 'manifest', namespace_id: group.id, path: project_path, diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb index ac3492dbe33..da3b597a74e 100644 --- a/lib/gitlab/markdown_cache.rb +++ b/lib/gitlab/markdown_cache.rb @@ -3,7 +3,7 @@ module Gitlab module MarkdownCache # Increment this number every time the renderer changes its output - CACHE_COMMONMARK_VERSION = 24 + CACHE_COMMONMARK_VERSION = 25 CACHE_COMMONMARK_VERSION_START = 10 BaseError = Class.new(StandardError) diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 5fed3d38d7c..7bd55cce363 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -3,7 +3,6 @@ module Gitlab module Metrics include Gitlab::Metrics::Prometheus - include Gitlab::Metrics::Methods EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.01, 0.1, 1].freeze @@ -81,25 +80,16 @@ module Gitlab real_time = (real_stop - real_start) cpu_time = cpu_stop - cpu_start - real_duration_seconds = fetch_histogram("gitlab_#{name}_real_duration_seconds".to_sym) do + trans.observe("gitlab_#{name}_real_duration_seconds".to_sym, real_time) do docstring "Measure #{name}" - base_labels Transaction::BASE_LABELS buckets EXECUTION_MEASUREMENT_BUCKETS end - real_duration_seconds.observe(trans.labels, real_time) - - cpu_duration_seconds = fetch_histogram("gitlab_#{name}_cpu_duration_seconds".to_sym) do + trans.observe("gitlab_#{name}_cpu_duration_seconds".to_sym, cpu_time) do docstring "Measure #{name}" - base_labels Transaction::BASE_LABELS buckets EXECUTION_MEASUREMENT_BUCKETS with_feature "prometheus_metrics_measure_#{name}_cpu_duration" end - cpu_duration_seconds.observe(trans.labels, cpu_time) - - trans.increment("#{name}_real_time", real_time.in_milliseconds, false) - trans.increment("#{name}_cpu_time", cpu_time.in_milliseconds, false) - trans.increment("#{name}_call_count", 1, false) retval end diff --git a/lib/gitlab/metrics/dashboard/cache.rb b/lib/gitlab/metrics/dashboard/cache.rb index a9ccf0fea9b..54b5250d209 100644 --- a/lib/gitlab/metrics/dashboard/cache.rb +++ b/lib/gitlab/metrics/dashboard/cache.rb @@ -9,34 +9,53 @@ module Gitlab CACHE_KEYS = 'all_cached_metric_dashboards' class << self - # Stores a dashboard in the cache, documenting the key - # so the cached can be cleared in bulk at another time. - def fetch(key) - register_key(key) + # This class method (Gitlab::Metrics::Dashboard::Cache.fetch) can be used + # when the key does not need to be deleted by `delete_all!`. + # For example, out of the box dashboard caches do not need to be deleted. + delegate :fetch, to: :"Rails.cache" - Rails.cache.fetch(key) { yield } - end + alias_method :for, :new + end + + def initialize(project) + @project = project + end + + # Stores a dashboard in the cache, documenting the key + # so the cache can be cleared in bulk at another time. + def fetch(key) + register_key(key) + + Rails.cache.fetch(key) { yield } + end - # Resets all dashboard caches, such that all - # dashboard content will be loaded from source on - # subsequent dashboard calls. - def delete_all! - all_keys.each { |key| Rails.cache.delete(key) } + # Resets all dashboard caches, such that all + # dashboard content will be loaded from source on + # subsequent dashboard calls. + def delete_all! + all_keys.each { |key| Rails.cache.delete(key) } - Rails.cache.delete(CACHE_KEYS) - end + Rails.cache.delete(catalog_key) + end - private + private - def register_key(key) - new_keys = all_keys.add(key).to_a.join('|') + def register_key(key) + new_keys = all_keys.add(key).to_a.join('|') - Rails.cache.write(CACHE_KEYS, new_keys) - end + Rails.cache.write(catalog_key, new_keys) + end + + def all_keys + keys = Rails.cache.read(catalog_key)&.split('|') + Set.new(keys) + end - def all_keys - Set.new(Rails.cache.read(CACHE_KEYS)&.split('|')) - end + # One key to store them all... + # This key is used to store the names of all the keys that contain this + # project's dashboards. + def catalog_key + "#{CACHE_KEYS}_#{@project.id}" end end end diff --git a/lib/gitlab/metrics/dashboard/defaults.rb b/lib/gitlab/metrics/dashboard/defaults.rb index 3c39a7c6911..6a5f98a18c8 100644 --- a/lib/gitlab/metrics/dashboard/defaults.rb +++ b/lib/gitlab/metrics/dashboard/defaults.rb @@ -7,7 +7,6 @@ module Gitlab module Dashboard module Defaults DEFAULT_PANEL_TYPE = 'area-chart' - DEFAULT_PANEL_WEIGHT = 0 end end end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index 5e2d78e10a4..2c4793eb75f 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -14,10 +14,7 @@ module Gitlab ::Metrics::Dashboard::SelfMonitoringDashboardService, # This dashboard is displayed on the K8s cluster settings health page. - ::Metrics::Dashboard::ClusterDashboardService, - - # This dashboard is not yet ready for the world. - ::Metrics::Dashboard::PodDashboardService + ::Metrics::Dashboard::ClusterDashboardService ].freeze class << self @@ -72,17 +69,11 @@ module Gitlab # display_name: String, # default: Boolean }] def find_all_paths(project) - project.repository.metrics_dashboard_paths - end - - # Summary of all known dashboards. Used to populate repo cache. - # Prefer #find_all_paths. - def find_all_paths_from_source(project) - Gitlab::Metrics::Dashboard::Cache.delete_all! - - user_facing_dashboard_services(project).flat_map do |service| + dashboards = user_facing_dashboard_services(project).flat_map do |service| service.all_dashboard_paths(project) end + + Gitlab::Utils.stable_sort_by(dashboards) { |dashboard| dashboard[:display_name].downcase } end private diff --git a/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb b/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb new file mode 100644 index 00000000000..8b791e110ba --- /dev/null +++ b/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# Provides methods to list and read dashboard yaml files from a project's repository. +module Gitlab + module Metrics + module Dashboard + class RepoDashboardFinder + DASHBOARD_ROOT = ".gitlab/dashboards" + DASHBOARD_EXTENSION = '.yml' + + class << self + # Returns list of all user-defined dashboard paths. Used to populate + # Repository model cache (Repository#user_defined_metrics_dashboard_paths). + # Also deletes all dashboard cache entries. + # @return [Array] ex) ['.gitlab/dashboards/dashboard1.yml'] + def list_dashboards(project) + Gitlab::Metrics::Dashboard::Cache.for(project).delete_all! + + file_finder(project).list_files_for(DASHBOARD_ROOT) + end + + # Reads the given dashboard from repository, and returns the content as a string. + # @return [String] + def read_dashboard(project, dashboard_path) + file_finder(project).read(dashboard_path) + end + + private + + def file_finder(project) + Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, DASHBOARD_EXTENSION) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb index 3444a01bccd..3b49eb1c837 100644 --- a/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb @@ -9,7 +9,10 @@ module Gitlab # config. If there are no project-specific metrics, # this will have no effect. def transform! - PrometheusMetricsFinder.new(project: project).execute.each do |project_metric| + custom_metrics = PrometheusMetricsFinder.new(project: project, ordered: true).execute + custom_metrics = Gitlab::Utils.stable_sort_by(custom_metrics) { |metric| -metric.priority } + + custom_metrics.each do |project_metric| group = find_or_create_panel_group(dashboard[:panel_groups], project_metric) panel = find_or_create_panel(group[:panels], project_metric) find_or_create_metric(panel[:metrics], project_metric) @@ -83,7 +86,6 @@ module Gitlab def new_panel_group(metric) { group: metric.group_title, - priority: metric.priority, panels: [] } end diff --git a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb index c48a7ff25a5..dd85bd0beb1 100644 --- a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb @@ -45,7 +45,9 @@ module Gitlab raise Errors::MissingQueryError.new('Each "metric" must define one of :query or :query_range') unless query - query + # We need to remove any newlines since our UrlBlocker does not allow + # multiline URLs. + query.to_s.squish end end end diff --git a/lib/gitlab/metrics/dashboard/stages/sorter.rb b/lib/gitlab/metrics/dashboard/stages/sorter.rb deleted file mode 100644 index 882211e1441..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/sorter.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class Sorter < BaseStage - def transform! - missing_panel_groups! unless dashboard[:panel_groups].is_a? Array - - sort_groups! - sort_panels! - end - - private - - # Sorts the groups in the dashboard by the :priority key - def sort_groups! - dashboard[:panel_groups] = Gitlab::Utils.stable_sort_by(dashboard[:panel_groups]) { |group| -group[:priority].to_i } - end - - # Sorts the panels in the dashboard by the :weight key - def sort_panels! - dashboard[:panel_groups].each do |group| - missing_panels! unless group[:panels].is_a? Array - - group[:panels] = Gitlab::Utils.stable_sort_by(group[:panels]) { |panel| -panel[:weight].to_i } - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb b/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb new file mode 100644 index 00000000000..71da779d16c --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class TrackPanelType < BaseStage + def transform! + for_panel_groups do |panel_group| + for_panels_in(panel_group) do |panel| + track_panel_type(panel) + end + end + end + + private + + def track_panel_type(panel) + panel_type = panel[:type] + + Gitlab::Tracking.event('MetricsDashboard::Chart', 'chart_rendered', label: panel_type) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb index 10a2f3c2397..160ecfb85c9 100644 --- a/lib/gitlab/metrics/dashboard/url.rb +++ b/lib/gitlab/metrics/dashboard/url.rb @@ -43,6 +43,39 @@ module Gitlab end end + # Matches dashboard urls for a metric chart embed + # for cluster metrics + # + # EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB) + def clusters_regex + strong_memoize(:clusters_regex) do + regex_for_project_metrics( + %r{ + /clusters + /(?<cluster_id>\d+) + /? + }x + ) + end + end + + # Matches dashboard urls for a metric chart embed + # for a specifc firing GitLab alert + # + # EX - https://<host>/<namespace>/<project>/prometheus/alerts/<alert_id>/metrics_dashboard + def alert_regex + strong_memoize(:alert_regex) do + regex_for_project_metrics( + %r{ + /prometheus + /alerts + /(?<alert>\d+) + /metrics_dashboard + }x + ) + end + end + # Parses query params out from full url string into hash. # # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group' @@ -60,22 +93,6 @@ module Gitlab Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args) end - # Matches dashboard urls for a metric chart embed - # for cluster metrics - # - # EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB) - def clusters_regex - strong_memoize(:clusters_regex) do - regex_for_project_metrics( - %r{ - /clusters - /(?<cluster_id>\d+) - /? - }x - ) - end - end - private def regex_for_project_metrics(path_suffix_pattern) @@ -92,16 +109,18 @@ module Gitlab end def gitlab_host_pattern - Regexp.escape(Gitlab.config.gitlab.url) + Regexp.escape(gitlab_domain) end def project_path_pattern "\/#{Project.reference_pattern}" end + + def gitlab_domain + Gitlab.config.gitlab.url + end end end end end end - -Gitlab::Metrics::Dashboard::Url.extend_if_ee('::EE::Gitlab::Metrics::Dashboard::Url') diff --git a/lib/gitlab/metrics/dashboard/validator.rb b/lib/gitlab/metrics/dashboard/validator.rb new file mode 100644 index 00000000000..8edd9c397f9 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Validator + DASHBOARD_SCHEMA_PATH = 'lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json'.freeze + + class << self + def validate(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) + errors = _validate(content, schema_path, dashboard_path: dashboard_path, project: project) + errors.empty? + end + + def validate!(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) + errors = _validate(content, schema_path, dashboard_path: dashboard_path, project: project) + errors.empty? || raise(errors.first) + end + + private + + def _validate(content, schema_path, dashboard_path: nil, project: nil) + client = Validator::Client.new(content, schema_path, dashboard_path: dashboard_path, project: project) + client.execute + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/validator/client.rb b/lib/gitlab/metrics/dashboard/validator/client.rb new file mode 100644 index 00000000000..c63415abcfc --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/client.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Validator + class Client + # @param content [Hash] Representing a raw, unprocessed + # dashboard object + # @param schema_path [String] Representing path to dashboard schema file + # @param dashboard_path[String] Representing path to dashboard content file + # @param project [Project] Project to validate dashboard against + def initialize(content, schema_path, dashboard_path: nil, project: nil) + @content = content + @schema_path = schema_path + @dashboard_path = dashboard_path + @project = project + end + + def execute + errors = validate_against_schema + errors += post_schema_validator.validate + + errors.compact + end + + private + + attr_reader :content, :schema_path, :project, :dashboard_path + + def custom_formats + @custom_formats ||= CustomFormats.new + end + + def post_schema_validator + PostSchemaValidator.new( + project: project, + metric_ids: custom_formats.metric_ids_cache, + dashboard_path: dashboard_path + ) + end + + def schemer + @schemer ||= ::JSONSchemer.schema(Pathname.new(schema_path), formats: custom_formats.format_handlers) + end + + def validate_against_schema + schemer.validate(content).map do |error| + Errors::SchemaValidationError.new(error) + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/validator/custom_formats.rb b/lib/gitlab/metrics/dashboard/validator/custom_formats.rb new file mode 100644 index 00000000000..485e80ad1b7 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/custom_formats.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Validator + class CustomFormats + def format_handlers + # Key is custom JSON Schema format name. Value is a proc that takes data and schema and handles + # validations. + @format_handlers ||= { + "add_to_metric_id_cache" => ->(data, schema) { metric_ids_cache << data } + } + end + + def metric_ids_cache + @metric_ids_cache ||= [] + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/validator/errors.rb b/lib/gitlab/metrics/dashboard/validator/errors.rb new file mode 100644 index 00000000000..0f6e687d291 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/errors.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Validator + module Errors + InvalidDashboardError = Class.new(StandardError) + + class SchemaValidationError < InvalidDashboardError + def initialize(error = {}) + super(error_message(error)) + end + + private + + def error_message(error) + if error.is_a?(Hash) && error.present? + pretty(error) + else + "Dashboard failed schema validation" + end + end + + # based on https://github.com/davishmcclurg/json_schemer/blob/master/lib/json_schemer/errors.rb + # with addition ability to translate error messages + def pretty(error) + data, data_pointer, type, schema = error.values_at('data', 'data_pointer', 'type', 'schema') + location = data_pointer.empty? ? 'root' : data_pointer + + case type + when 'required' + keys = error.fetch('details').fetch('missing_keys').join(', ') + _("%{location} is missing required keys: %{keys}") % { location: location, keys: keys } + when 'null', 'string', 'boolean', 'integer', 'number', 'array', 'object' + _("'%{data}' at %{location} is not of type: %{type}") % { data: data, location: location, type: type } + when 'pattern' + _("'%{data}' at %{location} does not match pattern: %{pattern}") % { data: data, location: location, pattern: schema.fetch('pattern') } + when 'format' + _("'%{data}' at %{location} does not match format: %{format}") % { data: data, location: location, format: schema.fetch('format') } + when 'const' + _("'%{data}' at %{location} is not: %{const}") % { data: data, location: location, const: schema.fetch('const').inspect } + when 'enum' + _("'%{data}' at %{location} is not one of: %{enum}") % { data: data, location: location, enum: schema.fetch('enum') } + else + _("'%{data}' at %{location} is invalid: error_type=%{type}") % { data: data, location: location, type: type } + end + end + end + + class DuplicateMetricIds < InvalidDashboardError + def initialize + super(_("metric_id must be unique across a project")) + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb b/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb new file mode 100644 index 00000000000..73bfc5a6294 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Validator + class PostSchemaValidator + def initialize(metric_ids:, project: nil, dashboard_path: nil) + @metric_ids = metric_ids + @project = project + @dashboard_path = dashboard_path + end + + def validate + errors = [] + errors << uniq_metric_ids + errors.compact + end + + private + + attr_reader :project, :metric_ids, :dashboard_path + + def uniq_metric_ids + return Validator::Errors::DuplicateMetricIds.new if metric_ids.uniq! + + uniq_metric_ids_across_project if project.present? || dashboard_path.present? + end + + # rubocop: disable CodeReuse/ActiveRecord + def uniq_metric_ids_across_project + return ArgumentError.new(_('Both project and dashboard_path are required')) unless + dashboard_path.present? && project.present? + + # If PrometheusMetric identifier is not unique across project and dashboard_path, + # we need to error because we don't know if the user is trying to create a new metric + # or update an existing one. + identifier_on_other_dashboard = PrometheusMetric.where( + project: project, + identifier: metric_ids + ).where.not( + dashboard_path: dashboard_path + ).exists? + + Validator::Errors::DuplicateMetricIds.new if identifier_on_other_dashboard + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/axis.json b/lib/gitlab/metrics/dashboard/validator/schemas/axis.json new file mode 100644 index 00000000000..54334022426 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/axis.json @@ -0,0 +1,14 @@ +{ + "type": "object", + "properties": { + "name": { "type": "string" }, + "format": { + "type": "string", + "default": "engineering" + }, + "precision": { + "type": "number", + "default": 2 + } + } +} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json b/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json new file mode 100644 index 00000000000..313f03be7dc --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json @@ -0,0 +1,18 @@ +{ + "type": "object", + "required": ["dashboard", "panel_groups"], + "properties": { + "dashboard": { "type": "string" }, + "panel_groups": { + "type": "array", + "items": { "$ref": "./panel_group.json" } + }, + "templating": { + "$ref": "./templating.json" + }, + "links": { + "type": "array", + "items": { "$ref": "./link.json" } + } + } +} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/link.json b/lib/gitlab/metrics/dashboard/validator/schemas/link.json new file mode 100644 index 00000000000..4ea7b5dd324 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/link.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "required": ["url"], + "properties": { + "url": { "type": "string" }, + "title": { "type": "string" }, + "type": { + "type": "string", + "enum": ["grafana"] + } + } +} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/metric.json b/lib/gitlab/metrics/dashboard/validator/schemas/metric.json new file mode 100644 index 00000000000..13831b77e3e --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/metric.json @@ -0,0 +1,16 @@ +{ + "type": "object", + "required": ["unit"], + "oneOf": [{ "required": ["query"] }, { "required": ["query_range"] }], + "properties": { + "id": { + "type": "string", + "format": "add_to_metric_id_cache" + }, + "unit": { "type": "string" }, + "label": { "type": "string" }, + "query": { "type": ["string", "number"] }, + "query_range": { "type": ["string", "number"] }, + "step": { "type": "number" } + } +} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/panel.json b/lib/gitlab/metrics/dashboard/validator/schemas/panel.json new file mode 100644 index 00000000000..011eef53e40 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/panel.json @@ -0,0 +1,24 @@ +{ + "type": "object", + "required": ["title", "metrics"], + "properties": { + "type": { + "type": "string", + "enum": ["area-chart", "anomaly-chart", "bar", "column", "stacked-column", "single-stat", "heatmap"], + "default": "area-chart" + }, + "title": { "type": "string" }, + "y_label": { "type": "string" }, + "y_axis": { "$ref": "./axis.json" }, + "max_value": { "type": "number" }, + "weight": { "type": "number" }, + "metrics": { + "type": "array", + "items": { "$ref": "./metric.json" } + }, + "links": { + "type": "array", + "items": { "$ref": "./link.json" } + } + } +} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json b/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json new file mode 100644 index 00000000000..1306fc475db --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "required": ["group", "panels"], + "properties": { + "group": { "type": "string" }, + "priority": { "type": "number" }, + "panels": { + "type": "array", + "items": { "$ref": "./panel.json" } + } + } +} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/templating.json b/lib/gitlab/metrics/dashboard/validator/schemas/templating.json new file mode 100644 index 00000000000..6f8664c89af --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/templating.json @@ -0,0 +1,7 @@ +{ + "type": "object", + "required": ["variables"], + "properties": { + "variables": { "type": "object" } + } +} diff --git a/lib/gitlab/metrics/elasticsearch_rack_middleware.rb b/lib/gitlab/metrics/elasticsearch_rack_middleware.rb index 6830eed68d5..870ab148004 100644 --- a/lib/gitlab/metrics/elasticsearch_rack_middleware.rb +++ b/lib/gitlab/metrics/elasticsearch_rack_middleware.rb @@ -4,18 +4,10 @@ module Gitlab module Metrics # Rack middleware for tracking Elasticsearch metrics from Grape and Web requests. class ElasticsearchRackMiddleware - HISTOGRAM_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60].freeze + HISTOGRAM_BUCKETS = [0.1, 0.5, 1, 10, 50].freeze def initialize(app) @app = app - - @requests_total_counter = Gitlab::Metrics.counter(:http_elasticsearch_requests_total, - 'Amount of calls to Elasticsearch servers during web requests', - Gitlab::Metrics::Transaction::BASE_LABELS) - @requests_duration_histogram = Gitlab::Metrics.histogram(:http_elasticsearch_requests_duration_seconds, - 'Query time for Elasticsearch servers during web requests', - Gitlab::Metrics::Transaction::BASE_LABELS, - HISTOGRAM_BUCKETS) end def call(env) @@ -29,12 +21,19 @@ module Gitlab private def record_metrics(transaction) - labels = transaction.labels query_time = ::Gitlab::Instrumentation::ElasticsearchTransport.query_time request_count = ::Gitlab::Instrumentation::ElasticsearchTransport.get_request_count - @requests_total_counter.increment(labels, request_count) - @requests_duration_histogram.observe(labels, query_time) + return unless request_count > 0 + + transaction.increment(:http_elasticsearch_requests_total, request_count) do + docstring 'Amount of calls to Elasticsearch servers during web requests' + end + + transaction.observe(:http_elasticsearch_requests_duration_seconds, query_time) do + docstring 'Query time for Elasticsearch servers during web requests' + buckets HISTOGRAM_BUCKETS + end end end end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index fbeda3b75e0..c6b0a0c5e76 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -4,16 +4,7 @@ module Gitlab module Metrics # Class for tracking timing information about method calls class MethodCall - include Gitlab::Metrics::Methods - BASE_LABELS = { module: nil, method: nil }.freeze - attr_reader :real_time, :cpu_time, :call_count, :labels - - define_histogram :gitlab_method_call_duration_seconds do - docstring 'Method calls real duration' - base_labels Transaction::BASE_LABELS.merge(BASE_LABELS) - buckets [0.01, 0.05, 0.1, 0.5, 1] - with_feature :prometheus_metrics_method_instrumentation - end + attr_reader :real_time, :cpu_time, :call_count # name - The full name of the method (including namespace) such as # `User#sign_in`. @@ -42,8 +33,14 @@ module Gitlab @cpu_time += cpu_time @call_count += 1 - if above_threshold? - self.class.gitlab_method_call_duration_seconds.observe(@transaction.labels.merge(labels), real_time) + if above_threshold? && transaction + label_keys = labels.keys + transaction.observe(:gitlab_method_call_duration_seconds, real_time, labels) do + docstring 'Method calls real duration' + label_keys label_keys + buckets [0.01, 0.05, 0.1, 0.5, 1] + with_feature :prometheus_metrics_method_instrumentation + end end retval @@ -54,6 +51,10 @@ module Gitlab def above_threshold? real_time.in_milliseconds >= ::Gitlab::Metrics.method_call_threshold end + + private + + attr_reader :labels, :transaction end end end diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index 83a7b925392..2b5d1c710f6 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -69,62 +69,6 @@ module Gitlab raise ArgumentError, "uknown metric type #{type}" end end - - # Fetch and/or initialize counter metric - # @param [Symbol] name - # @param [Hash] opts - def fetch_counter(name, opts = {}, &block) - fetch_metric(:counter, name, opts, &block) - end - - # Fetch and/or initialize gauge metric - # @param [Symbol] name - # @param [Hash] opts - def fetch_gauge(name, opts = {}, &block) - fetch_metric(:gauge, name, opts, &block) - end - - # Fetch and/or initialize histogram metric - # @param [Symbol] name - # @param [Hash] opts - def fetch_histogram(name, opts = {}, &block) - fetch_metric(:histogram, name, opts, &block) - end - - # Fetch and/or initialize summary metric - # @param [Symbol] name - # @param [Hash] opts - def fetch_summary(name, opts = {}, &block) - fetch_metric(:summary, name, opts, &block) - end - - # Define metric accessor method for a Counter - # @param [Symbol] name - # @param [Hash] opts - def define_counter(name, opts = {}, &block) - define_metric(:counter, name, opts, &block) - end - - # Define metric accessor method for a Gauge - # @param [Symbol] name - # @param [Hash] opts - def define_gauge(name, opts = {}, &block) - define_metric(:gauge, name, opts, &block) - end - - # Define metric accessor method for a Histogram - # @param [Symbol] name - # @param [Hash] opts - def define_histogram(name, opts = {}, &block) - define_metric(:histogram, name, opts, &block) - end - - # Define metric accessor method for a Summary - # @param [Symbol] name - # @param [Hash] opts - def define_summary(name, opts = {}, &block) - define_metric(:summary, name, opts, &block) - end end end end diff --git a/lib/gitlab/metrics/methods/metric_options.rb b/lib/gitlab/metrics/methods/metric_options.rb index 8e6ceb74c09..1e488df3e99 100644 --- a/lib/gitlab/metrics/methods/metric_options.rb +++ b/lib/gitlab/metrics/methods/metric_options.rb @@ -4,14 +4,12 @@ module Gitlab module Metrics module Methods class MetricOptions - SMALL_NETWORK_BUCKETS = [0.005, 0.01, 0.1, 1, 10].freeze - def initialize(options = {}) @multiprocess_mode = options[:multiprocess_mode] || :all - @buckets = options[:buckets] || SMALL_NETWORK_BUCKETS - @base_labels = options[:base_labels] || {} + @buckets = options[:buckets] || ::Prometheus::Client::Histogram::DEFAULT_BUCKETS @docstring = options[:docstring] @with_feature = options[:with_feature] + @label_keys = options[:label_keys] || [] end # Documentation describing metric in metrics endpoint '/-/metrics' @@ -40,12 +38,21 @@ module Gitlab end # Base labels are merged with per metric labels - def base_labels(base_labels = nil) - @base_labels = base_labels unless base_labels.nil? + def base_labels + @base_labels ||= @label_keys.product([nil]).to_h @base_labels end + def label_keys(label_keys = nil) + unless label_keys.nil? + @label_keys = label_keys + @base_labels = nil + end + + @label_keys + end + # Use feature toggle to control whether certain metric is enabled/disabled def with_feature(name = nil) @with_feature = name unless name.nil? @@ -55,6 +62,7 @@ module Gitlab def evaluate(&block) instance_eval(&block) if block_given? + self end end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index c6a0457ffe5..a6884ea6983 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -10,8 +10,7 @@ module Gitlab # env - A Hash containing Rack environment details. def call(env) - trans = transaction_from_env(env) - retval = nil + trans = WebTransaction.new(env) begin retval = trans.run { @app.call(env) } @@ -24,21 +23,6 @@ module Gitlab retval end - - def transaction_from_env(env) - trans = WebTransaction.new(env) - - trans.set(:request_uri, filtered_path(env), false) - trans.set(:request_method, env['REQUEST_METHOD'], false) - - trans - end - - private - - def filtered_path(env) - ActionDispatch::Request.new(env).filtered_path.presence || env['REQUEST_URI'] - end end end end diff --git a/lib/gitlab/metrics/redis_rack_middleware.rb b/lib/gitlab/metrics/redis_rack_middleware.rb deleted file mode 100644 index f0f99c5f45d..00000000000 --- a/lib/gitlab/metrics/redis_rack_middleware.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - # Rack middleware for tracking Redis metrics from Grape and Web requests. - class RedisRackMiddleware - def initialize(app) - @app = app - - @requests_total_counter = Gitlab::Metrics.counter(:http_redis_requests_total, - 'Amount of calls to Redis servers during web requests', - Gitlab::Metrics::Transaction::BASE_LABELS) - @requests_duration_histogram = Gitlab::Metrics.histogram(:http_redis_requests_duration_seconds, - 'Query time for Redis servers during web requests', - Gitlab::Metrics::Transaction::BASE_LABELS, - Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS) - end - - def call(env) - transaction = Gitlab::Metrics.current_transaction - - @app.call(env) - ensure - record_metrics(transaction) - end - - private - - def record_metrics(transaction) - labels = transaction.labels - query_time = Gitlab::Instrumentation::Redis.query_time - request_count = Gitlab::Instrumentation::Redis.get_request_count - - @requests_total_counter.increment(labels, request_count) - @requests_duration_histogram.observe(labels, query_time) - end - end - end -end diff --git a/lib/gitlab/metrics/samplers/threads_sampler.rb b/lib/gitlab/metrics/samplers/threads_sampler.rb new file mode 100644 index 00000000000..05acef7ce0c --- /dev/null +++ b/lib/gitlab/metrics/samplers/threads_sampler.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Samplers + class ThreadsSampler < BaseSampler + SAMPLING_INTERVAL_SECONDS = 5 + KNOWN_PUMA_THREAD_NAMES = ['puma worker check pipe', 'puma server', + 'puma threadpool reaper', 'puma threadpool trimmer', + 'puma worker check pipe', 'puma stat payload'].freeze + + SIDEKIQ_WORKER_THREAD_NAME = 'sidekiq_worker_thread' + + METRIC_PREFIX = "gitlab_ruby_threads_" + + METRIC_DESCRIPTIONS = { + max_expected_threads: "Maximum number of threads expected to be running and performing application work", + running_threads: "Number of running Ruby threads by name" + }.freeze + + def metrics + @metrics ||= METRIC_DESCRIPTIONS.each_with_object({}) do |(name, description), result| + result[name] = ::Gitlab::Metrics.gauge(:"#{METRIC_PREFIX}#{name}", description) + end + end + + def sample + metrics[:max_expected_threads].set({}, Gitlab::Runtime.max_threads) + + threads_by_name.each do |name, threads| + uses_db, not_using_db = threads.partition { |thread| thread[:uses_db_connection] } + + set_running_threads(name, uses_db_connection: "yes", size: uses_db.size) + set_running_threads(name, uses_db_connection: "no", size: not_using_db.size) + end + end + + private + + def set_running_threads(name, uses_db_connection:, size:) + metrics[:running_threads].set({ thread_name: name, uses_db_connection: uses_db_connection }, size) + end + + def threads_by_name + Thread.list.group_by { |thread| name_for_thread(thread) } + end + + def uses_db_connection(thread) + thread[:uses_db_connection] ? "yes" : "no" + end + + def name_for_thread(thread) + thread_name = thread.name.to_s.presence + + if thread_name.presence.nil? + 'unnamed' + elsif thread_name =~ /puma threadpool \d+/ + # These are the puma workers processing requests + 'puma threadpool' + elsif use_thread_name?(thread_name) + thread_name + else + 'unrecognized' + end + end + + def use_thread_name?(thread_name) + thread_name == SIDEKIQ_WORKER_THREAD_NAME || + # Samplers defined in `lib/gitlab/metrics/samplers` + thread_name.ends_with?('sampler') || + # Exporters from `lib/gitlab/metrics/exporter` + thread_name.ends_with?('exporter') || + KNOWN_PUMA_THREAD_NAMES.include?(thread_name) + end + end + end + end +end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index 1c99e1e730c..8c4e5a8d70c 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -12,7 +12,9 @@ module Gitlab begin # Old gitlad-shell messages don't provide enqueued_at/created_at attributes enqueued_at = payload['enqueued_at'] || payload['created_at'] || 0 - trans.set(:sidekiq_queue_duration, Time.current.to_f - enqueued_at) + trans.set(:gitlab_transaction_sidekiq_queue_duration_total, Time.current.to_f - enqueued_at) do + multiprocess_mode :livesum + end trans.run { yield } rescue Exception => error # rubocop: disable Lint/RescueException trans.add_event(:sidekiq_exception) diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 24107e42aa9..e1f1f37c905 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -5,14 +5,6 @@ module Gitlab module Subscribers # Class for tracking the rendering timings of views. class ActionView < ActiveSupport::Subscriber - include Gitlab::Metrics::Methods - define_histogram :gitlab_view_rendering_duration_seconds do - docstring 'View rendering time' - base_labels Transaction::BASE_LABELS.merge({ path: nil }) - buckets [0.001, 0.01, 0.1, 1, 10.0] - with_feature :prometheus_metrics_view_instrumentation - end - attach_to :action_view SERIES = 'views' @@ -27,10 +19,14 @@ module Gitlab def track(event) tags = tags_for(event) - - self.class.gitlab_view_rendering_duration_seconds.observe(current_transaction.labels.merge(tags), event.duration) - - current_transaction.increment(:view_duration, event.duration) + current_transaction.observe(:gitlab_view_rendering_duration_seconds, event.duration, tags) do + docstring 'View rendering time' + label_keys %i(view) + buckets [0.001, 0.01, 0.1, 1, 10.0] + with_feature :prometheus_metrics_view_instrumentation + end + + current_transaction.increment(:gitlab_transaction_view_duration_total, event.duration) end def relative_path(path) diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index d2736882432..e53ac00e77f 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -5,20 +5,25 @@ module Gitlab module Subscribers # Class for tracking the total query duration of a transaction. class ActiveRecord < ActiveSupport::Subscriber - include Gitlab::Metrics::Methods attach_to :active_record IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze def sql(event) + # Mark this thread as requiring a database connection. This is used + # by the Gitlab::Metrics::Samplers::ThreadsSampler to count threads + # using a connection. + Thread.current[:uses_db_connection] = true + return unless current_transaction payload = event.payload - return if payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql]) - self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) + current_transaction.observe(:gitlab_sql_duration_seconds, event.duration / 1000.0) do + buckets [0.05, 0.1, 0.25] + end increment_db_counters(payload) end @@ -33,12 +38,6 @@ module Gitlab private - define_histogram :gitlab_sql_duration_seconds do - docstring 'SQL time' - base_labels Transaction::BASE_LABELS - buckets [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0] - end - def select_sql_command?(payload) payload[:sql].match(/\A((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i) end @@ -54,7 +53,7 @@ module Gitlab end def increment(counter) - current_transaction.increment(counter, 1) + current_transaction.increment("gitlab_transaction_#{counter}_total".to_sym, 1) if Gitlab::SafeRequestStore.active? Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index 2ee7144fe2f..b274d2b1079 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -14,11 +14,10 @@ module Gitlab return unless current_transaction return if event.payload[:super_operation] == :fetch - if event.payload[:hit] - current_transaction.increment(:cache_read_hit_count, 1, false) - else - metric_cache_misses_total.increment(current_transaction.labels) - current_transaction.increment(:cache_read_miss_count, 1, false) + unless event.payload[:hit] + current_transaction.increment(:gitlab_cache_misses_total, 1) do + docstring 'Cache read miss' + end end end @@ -37,25 +36,30 @@ module Gitlab def cache_fetch_hit(event) return unless current_transaction - current_transaction.increment(:cache_read_hit_count, 1) + current_transaction.increment(:gitlab_transaction_cache_read_hit_count_total, 1) end def cache_generate(event) return unless current_transaction - metric_cache_misses_total.increment(current_transaction.labels) - current_transaction.increment(:cache_read_miss_count, 1) + current_transaction.increment(:gitlab_cache_misses_total, 1) do + docstring 'Cache read miss' + end + + current_transaction.increment(:gitlab_transaction_cache_read_miss_count_total, 1) end def observe(key, duration) return unless current_transaction - metric_cache_operations_total.increment(current_transaction.labels.merge({ operation: key })) - metric_cache_operation_duration_seconds.observe({ operation: key }, duration / 1000.0) - current_transaction.increment(:cache_duration, duration, false) - current_transaction.increment(:cache_count, 1, false) - current_transaction.increment("cache_#{key}_duration".to_sym, duration, false) - current_transaction.increment("cache_#{key}_count".to_sym, 1, false) + labels = { operation: key } + + current_transaction.increment(:gitlab_cache_operations_total, 1, labels) do + docstring 'Cache operations' + label_keys labels.keys + end + + metric_cache_operation_duration_seconds.observe(labels, duration / 1000.0) end private @@ -64,14 +68,6 @@ module Gitlab Transaction.current end - def metric_cache_operations_total - @metric_cache_operations_total ||= ::Gitlab::Metrics.counter( - :gitlab_cache_operations_total, - 'Cache operations', - Transaction::BASE_LABELS - ) - end - def metric_cache_operation_duration_seconds @metric_cache_operation_duration_seconds ||= ::Gitlab::Metrics.histogram( :gitlab_cache_operation_duration_seconds, @@ -80,14 +76,6 @@ module Gitlab [0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0] ) end - - def metric_cache_misses_total - @metric_cache_misses_total ||= ::Gitlab::Metrics.counter( - :gitlab_cache_misses_total, - 'Cache read miss', - Transaction::BASE_LABELS - ) - end end end end diff --git a/lib/gitlab/metrics/templates/Area.metrics-dashboard.yml b/lib/gitlab/metrics/templates/Area.metrics-dashboard.yml new file mode 100644 index 00000000000..1f7dd25aaee --- /dev/null +++ b/lib/gitlab/metrics/templates/Area.metrics-dashboard.yml @@ -0,0 +1,15 @@ +# Only one dashboard should be defined per file +# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html +dashboard: 'Area Panel Example' + +# For more information about the required properties of panel_groups +# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties +panel_groups: + - group: 'Server Statistics' + panels: + - title: Average amount of time spent by the CPU + type: area-chart + metrics: + - query_range: 'rate(node_cpu_seconds_total[15m])' + unit: 'Seconds' + label: "Time in Seconds" diff --git a/lib/gitlab/metrics/templates/Default.metrics-dashboard.yml b/lib/gitlab/metrics/templates/Default.metrics-dashboard.yml new file mode 100644 index 00000000000..b331e792461 --- /dev/null +++ b/lib/gitlab/metrics/templates/Default.metrics-dashboard.yml @@ -0,0 +1,24 @@ +# Only one dashboard should be defined per file +# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html +dashboard: 'Single Stat' + +# This is where all of the variables that can be manipulated via the UI +# are initialized +# Check out: https://docs.gitlab.com/ee/operations/metrics/dashboards/templating_variables.html#templating-variables-for-metrics-dashboards-core +templating: + variables: + job: 'prometheus' + +# For more information about the required properties of panel_groups +# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties +panel_groups: + - group: 'Memory' + panels: + - title: Prometheus + type: single-stat + metrics: + # Queries that make use of variables need to have double curly brackets {} + # set to the variables, per the example below + - query: 'max(go_memstats_alloc_bytes{job="{{job}}"}) / 1024 /1024' + unit: '%' + label: "Max" diff --git a/lib/gitlab/metrics/templates/gauge.metrics-dashboard.yml b/lib/gitlab/metrics/templates/gauge.metrics-dashboard.yml new file mode 100644 index 00000000000..1c17a3a4d40 --- /dev/null +++ b/lib/gitlab/metrics/templates/gauge.metrics-dashboard.yml @@ -0,0 +1,23 @@ +# Only one dashboard should be defined per file +# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html +dashboard: 'Gauge Panel Example' + +# For more information about the required properties of panel_groups +# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties +panel_groups: + - group: 'Server Statistics' + panels: + - title: "Memory usage" + # More information about gauge panel types can be found here: + # https://docs.gitlab.com/ee/operations/metrics/dashboards/panel_types.html#gauge + type: "gauge-chart" + min_value: 0 + max_value: 1024 + split: 10 + thresholds: + mode: "percentage" + values: [60, 90] + format: "megabytes" + metrics: + - query: '(node_memory_MemTotal_bytes - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes)) / 1024 / 1024' + unit: 'MB' diff --git a/lib/gitlab/metrics/templates/index.md b/lib/gitlab/metrics/templates/index.md new file mode 100644 index 00000000000..59fc85899da --- /dev/null +++ b/lib/gitlab/metrics/templates/index.md @@ -0,0 +1,3 @@ +# Development guide for Metrics Dashboard templates + +Please follow [the development guideline](../../../../doc/development/operations/metrics/templates.md) diff --git a/lib/gitlab/metrics/templates/k8s_area.metrics-dashboard.yml b/lib/gitlab/metrics/templates/k8s_area.metrics-dashboard.yml new file mode 100644 index 00000000000..aea816658d0 --- /dev/null +++ b/lib/gitlab/metrics/templates/k8s_area.metrics-dashboard.yml @@ -0,0 +1,15 @@ +# Only one dashboard should be defined per file +# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html +dashboard: 'Area Panel Example' + +# For more information about the required properties of panel_groups +# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties +panel_groups: + - group: 'Server Statistics' + panels: + - title: "Core Usage (Pod Average)" + type: area-chart + metrics: + - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (pod)) OR avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (pod_name))' + unit: 'cores' + label: "Pod Average (in seconds)" diff --git a/lib/gitlab/metrics/templates/k8s_gauge.metrics-dashboard.yml b/lib/gitlab/metrics/templates/k8s_gauge.metrics-dashboard.yml new file mode 100644 index 00000000000..7f97719765b --- /dev/null +++ b/lib/gitlab/metrics/templates/k8s_gauge.metrics-dashboard.yml @@ -0,0 +1,23 @@ +# Only one dashboard should be defined per file +# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html +dashboard: 'Gauge K8s Panel Example' + +# For more information about the required properties of panel_groups +# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties +panel_groups: + - group: 'Server Statistics' + panels: + - title: "Memory usage" + # More information about gauge panel types can be found here: + # https://docs.gitlab.com/ee/operations/metrics/dashboards/panel_types.html#gauge + type: "gauge-chart" + min_value: 0 + max_value: 1024 + split: 10 + thresholds: + mode: "percentage" + values: [60, 90] + format: "megabytes" + metrics: + - query: 'avg(sum(container_memory_usage_bytes{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024 OR avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024' + unit: 'MB' diff --git a/lib/gitlab/metrics/templates/k8s_single-stat.metrics-dashboard.yml b/lib/gitlab/metrics/templates/k8s_single-stat.metrics-dashboard.yml new file mode 100644 index 00000000000..829e12357ff --- /dev/null +++ b/lib/gitlab/metrics/templates/k8s_single-stat.metrics-dashboard.yml @@ -0,0 +1,17 @@ +# Only one dashboard should be defined per file +# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html +dashboard: 'Single Stat Panel Example' + +# For more information about the required properties of panel_groups +# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties +panel_groups: + - group: 'Server Statistics' + panels: + - title: "Memory usage" + # More information about heatmap panel types can be found here: + # https://docs.gitlab.com/ee/operations/metrics/dashboards/panel_types.html#single-stat + type: "single-stat" + metrics: + - query: 'avg(sum(container_memory_usage_bytes{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024 OR avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024' + unit: 'MB' + label: "Used memory" diff --git a/lib/gitlab/metrics/templates/single-stat.metrics-dashboard.yml b/lib/gitlab/metrics/templates/single-stat.metrics-dashboard.yml new file mode 100644 index 00000000000..18c27fffc7c --- /dev/null +++ b/lib/gitlab/metrics/templates/single-stat.metrics-dashboard.yml @@ -0,0 +1,17 @@ +# Only one dashboard should be defined per file +# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html +dashboard: 'Heatmap Panel Example' + +# For more information about the required properties of panel_groups +# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties +panel_groups: + - group: 'Server Statistics' + panels: + - title: "Memory usage" + # More information about heatmap panel types can be found here: + # https://docs.gitlab.com/ee/operations/metrics/dashboards/panel_types.html#single-stat + type: "single-stat" + metrics: + - query: '(node_memory_MemTotal_bytes - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes)) / 1024 / 1024' + unit: 'MB' + label: "Used memory" diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index da06be9c79c..95bc90f9dad 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -6,20 +6,35 @@ module Gitlab class Transaction include Gitlab::Metrics::Methods - # base labels shared among all transactions - BASE_LABELS = { controller: nil, action: nil, feature_category: nil }.freeze + # base label keys shared among all transactions + BASE_LABEL_KEYS = %i(controller action feature_category).freeze # labels that potentially contain sensitive information and will be filtered - FILTERED_LABELS = [:branch, :path].freeze + FILTERED_LABEL_KEYS = %i(branch path).freeze THREAD_KEY = :_gitlab_metrics_transaction + SMALL_BUCKETS = [0.1, 0.25, 0.5, 1.0, 2.5, 5.0].freeze + # The series to store events (e.g. Git pushes) in. EVENT_SERIES = 'events' attr_reader :method - def self.current - Thread.current[THREAD_KEY] + class << self + def current + Thread.current[THREAD_KEY] + end + + def prometheus_metric(name, type, &block) + fetch_metric(type, name) do + # set default metric options + docstring "#{name.to_s.humanize} #{type}" + + evaluate(&block) + # always filter sensitive labels and merge with base ones + label_keys BASE_LABEL_KEYS | (label_keys - FILTERED_LABEL_KEYS) + end + end end def initialize @@ -27,9 +42,6 @@ module Gitlab @started_at = nil @finished_at = nil - - @memory_before = 0 - @memory_after = 0 end def duration @@ -40,25 +52,22 @@ module Gitlab System.thread_cpu_duration(@thread_cputime_start) end - def allocated_memory - @memory_after - @memory_before - end - def run Thread.current[THREAD_KEY] = self - @memory_before = System.memory_usage_rss @started_at = System.monotonic_time @thread_cputime_start = System.thread_cpu_time yield ensure - @memory_after = System.memory_usage_rss @finished_at = System.monotonic_time - self.class.gitlab_transaction_cputime_seconds.observe(labels, thread_cpu_duration) - self.class.gitlab_transaction_duration_seconds.observe(labels, duration) - self.class.gitlab_transaction_allocated_memory_bytes.observe(labels, allocated_memory * 1024.0) + observe(:gitlab_transaction_cputime_seconds, thread_cpu_duration) do + buckets SMALL_BUCKETS + end + observe(:gitlab_transaction_duration_seconds, duration) do + buckets SMALL_BUCKETS + end Thread.current[THREAD_KEY] = nil end @@ -71,8 +80,12 @@ module Gitlab # event_name - The name of the event (e.g. "git_push"). # tags - A set of tags to attach to the event. def add_event(event_name, tags = {}) - filtered_tags = filter_tags(tags) - self.class.transaction_metric(event_name, :counter, prefix: 'event_', tags: filtered_tags).increment(filtered_tags.merge(labels)) + event_name = "gitlab_transaction_event_#{event_name}_total".to_sym + metric = self.class.prometheus_metric(event_name, :counter) do + label_keys tags.keys + end + + metric.increment(filter_labels(tags)) end # Returns a MethodCall object for the given name. @@ -84,52 +97,70 @@ module Gitlab method end - def increment(name, value, use_prometheus = true) - self.class.transaction_metric(name, :counter).increment(labels, value) if use_prometheus - end + # Increment counter metric + # + # It will initialize the metric if metric is not found + # + # block - if provided can be used to initialize metric with custom options (docstring, labels, with_feature) + # + # Example: + # ``` + # transaction.increment(:mestric_name, 1, { docstring: 'Custom title', base_labels: {sane: 'yes'} } ) do + # + # transaction.increment(:mestric_name, 1) do + # docstring 'Custom title' + # label_keys %i(sane) + # end + # ``` + def increment(name, value = 1, labels = {}, &block) + counter = self.class.prometheus_metric(name, :counter, &block) - def set(name, value, use_prometheus = true) - self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus + counter.increment(filter_labels(labels), value) end - def labels - BASE_LABELS - end + # Set gauge metric + # + # It will initialize the metric if metric is not found + # + # block - if provided, it can be used to initialize metric with custom options (docstring, labels, with_feature, multiprocess_mode) + # - multiprocess_mode is :all by default + # + # Example: + # ``` + # transaction.set(:mestric_name, 1) do + # multiprocess_mode :livesum + # end + # ``` + def set(name, value, labels = {}, &block) + gauge = self.class.prometheus_metric(name, :gauge, &block) - define_histogram :gitlab_transaction_cputime_seconds do - docstring 'Transaction thread cputime' - base_labels BASE_LABELS - buckets [0.1, 0.25, 0.5, 1.0, 2.5, 5.0] + gauge.set(filter_labels(labels), value) end - define_histogram :gitlab_transaction_duration_seconds do - docstring 'Transaction duration' - base_labels BASE_LABELS - buckets [0.1, 0.25, 0.5, 1.0, 2.5, 5.0] - end + # Observe histogram metric + # + # It will initialize the metric if metric is not found + # + # block - if provided, it can be used to initialize metric with custom options (docstring, labels, with_feature, buckets) + # + # Example: + # ``` + # transaction.observe(:mestric_name, 1) do + # buckets [100, 1000, 10000, 100000, 1000000, 10000000] + # end + # ``` + def observe(name, value, labels = {}, &block) + histogram = self.class.prometheus_metric(name, :histogram, &block) - define_histogram :gitlab_transaction_allocated_memory_bytes do - docstring 'Transaction allocated memory bytes' - base_labels BASE_LABELS - buckets [100, 1000, 10000, 100000, 1000000, 10000000] + histogram.observe(filter_labels(labels), value) end - def self.transaction_metric(name, type, prefix: nil, tags: {}) - metric_name = "gitlab_transaction_#{prefix}#{name}_total".to_sym - fetch_metric(type, metric_name) do - docstring "Transaction #{prefix}#{name} #{type}" - base_labels tags.merge(BASE_LABELS) - - if type == :gauge - multiprocess_mode :livesum - end - end + def labels + BASE_LABEL_KEYS.product([nil]).to_h end - private - - def filter_tags(tags) - tags.without(*FILTERED_LABELS) + def filter_labels(labels) + labels.empty? ? self.labels : labels.without(*FILTERED_LABEL_KEYS).merge(self.labels) end end end diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index 630788f1a8a..ce1065c0cb3 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -19,25 +19,19 @@ module Gitlab if trans && proxy_start # Time in milliseconds since gitlab-workhorse started the request duration = Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000 - trans.set(:rails_queue_duration, duration) + trans.set(:gitlab_transaction_rails_queue_duration_total, duration) do + multiprocess_mode :livesum + end duration_s = Gitlab::Utils.ms_to_round_sec(duration) - metric_rails_queue_duration_seconds.observe(trans.labels, duration_s) + trans.observe(:gitlab_rails_queue_duration_seconds, duration_s) do + docstring 'Measures latency between GitLab Workhorse forwarding a request to Rails' + end env[GITLAB_RAILS_QUEUE_DURATION_KEY] = duration_s end @app.call(env) end - - private - - def metric_rails_queue_duration_seconds - @metric_rails_queue_duration_seconds ||= Gitlab::Metrics.histogram( - :gitlab_rails_queue_duration_seconds, - 'Measures latency between GitLab Workhorse forwarding a request to Rails', - Gitlab::Metrics::Transaction::BASE_LABELS - ) - end end end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 1c49379e8d2..6573506e926 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -136,7 +136,7 @@ module Gitlab end def graphql_query? - request.post? && request.path.start_with?(GRAPHQL_URL) + request.post? && request.path.start_with?(File.join(relative_url, GRAPHQL_URL)) end end end diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index 33e0c6aa9b7..002171854ad 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -34,7 +34,7 @@ module Gitlab @second_collection_pages ||= Hash.new do |hash, page| second_collection_page = page - first_collection_page_count - offset = if second_collection_page < 1 || first_collection_page_count.zero? + offset = if second_collection_page < 1 || first_collection_page_count == 0 0 else per_page - first_collection_last_page_size diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb index c8cb8b6e020..33e709360ad 100644 --- a/lib/gitlab/pages.rb +++ b/lib/gitlab/pages.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab - class Pages + module Pages VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze INTERNAL_API_REQUEST_HEADER = 'Gitlab-Pages-Api-Request'.freeze MAX_SIZE = 1.terabyte diff --git a/lib/gitlab/pages/settings.rb b/lib/gitlab/pages/settings.rb new file mode 100644 index 00000000000..e3dbeee7b13 --- /dev/null +++ b/lib/gitlab/pages/settings.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + class Settings < ::SimpleDelegator + DiskAccessDenied = Class.new(StandardError) + + def path + if ::Gitlab::Runtime.web_server? && ENV['GITLAB_PAGES_DENY_DISK_ACCESS'] == '1' + begin + raise DiskAccessDenied + rescue DiskAccessDenied => ex + ::Gitlab::ErrorTracking.track_exception(ex) + end + end + + super + end + end + end +end diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb new file mode 100644 index 00000000000..651e3d5a807 --- /dev/null +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + class GitalyKeysetPager + attr_reader :request_context, :project + delegate :params, to: :request_context + + def initialize(request_context, project) + @request_context = request_context + @project = project + end + + # It is expected that the given finder will respond to `execute` method with `gitaly_pagination: true` option + # and supports pagination via gitaly. + def paginate(finder) + return paginate_via_gitaly(finder) if keyset_pagination_enabled? + + branches = ::Kaminari.paginate_array(finder.execute) + Gitlab::Pagination::OffsetPagination + .new(request_context) + .paginate(branches) + end + + private + + def keyset_pagination_enabled? + Feature.enabled?(:branch_list_keyset_pagination, project) && params[:pagination] == 'keyset' + end + + def paginate_via_gitaly(finder) + finder.execute(gitaly_pagination: true).tap do |records| + apply_headers(records) + end + end + + def apply_headers(records) + if records.count == params[:per_page] + Gitlab::Pagination::Keyset::HeaderBuilder + .new(request_context) + .add_next_page_header( + query_params_for(records.last) + ) + end + end + + def query_params_for(record) + # NOTE: page_token is name for now, but it could be dynamic if we have other gitaly finders + # that is based on something other than name + { page_token: record.name } + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/header_builder.rb b/lib/gitlab/pagination/keyset/header_builder.rb new file mode 100644 index 00000000000..69c468207f6 --- /dev/null +++ b/lib/gitlab/pagination/keyset/header_builder.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class HeaderBuilder + attr_reader :request_context + delegate :params, :header, :request, to: :request_context + + def initialize(request_context) + @request_context = request_context + end + + def add_next_page_header(query_params) + link = next_page_link(page_href(query_params)) + header('Links', link) + header('Link', link) + end + + private + + def next_page_link(href) + %(<#{href}>; rel="next") + end + + def page_href(query_params) + base_request_uri.tap do |uri| + uri.query = updated_params(query_params).to_query + end.to_s + end + + def base_request_uri + @base_request_uri ||= URI.parse(request.url).tap do |uri| + uri.host = Gitlab.config.gitlab.host + uri.port = Gitlab.config.gitlab.port + end + end + + def updated_params(query_params) + params.merge(query_params) + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/request_context.rb b/lib/gitlab/pagination/keyset/request_context.rb index 070fa844347..ba17fb03681 100644 --- a/lib/gitlab/pagination/keyset/request_context.rb +++ b/lib/gitlab/pagination/keyset/request_context.rb @@ -24,9 +24,11 @@ module Gitlab end def apply_headers(next_page) - link = pagination_links(next_page) - request.header('Links', link) - request.header('Link', link) + Gitlab::Pagination::Keyset::HeaderBuilder + .new(request) + .add_next_page_header( + query_params_for(next_page) + ) end private @@ -63,25 +65,8 @@ module Gitlab end end - def page_href(page) - base_request_uri.tap do |uri| - uri.query = query_params_for(page).to_query - end.to_s - end - - def pagination_links(next_page) - %(<#{page_href(next_page)}>; rel="next") - end - - def base_request_uri - @base_request_uri ||= URI.parse(request.request.url).tap do |uri| - uri.host = Gitlab.config.gitlab.host - uri.port = Gitlab.config.gitlab.port - end - end - def query_params_for(page) - request.params.merge(lower_bounds_params(page)) + lower_bounds_params(page) end end end diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb index e286c3d467e..2c95a9e8d91 100644 --- a/lib/gitlab/polling_interval.rb +++ b/lib/gitlab/polling_interval.rb @@ -20,7 +20,7 @@ module Gitlab end def self.polling_enabled? - !Gitlab::CurrentSettings.polling_interval_multiplier.zero? + Gitlab::CurrentSettings.polling_interval_multiplier != 0 end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index e6b25e71eb3..f8141278e48 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -81,7 +81,7 @@ module Gitlab counts = %i(limited_milestones_count limited_notes_count limited_merge_requests_count limited_issues_count limited_blobs_count wiki_blobs_count) - counts.all? { |count_method| public_send(count_method).zero? } # rubocop:disable GitlabSecurity/PublicSend + counts.all? { |count_method| public_send(count_method) == 0 } # rubocop:disable GitlabSecurity/PublicSend end private diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 69499b5494e..56e1154a672 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -77,12 +77,12 @@ module Gitlab # metric labels to their respective values. # # @return [Hash] mapping labels to their aggregate numeric values, or the empty hash if no results were found - def aggregate(aggregate_query, time: Time.now) + def aggregate(aggregate_query, time: Time.now, transform_value: :to_f) response = query(aggregate_query, time: time) response.to_h do |result| key = block_given? ? yield(result['metric']) : result['metric'] _timestamp, value = result['value'] - [key, value.to_i] + [key, value.public_send(transform_value)] # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index 609087d8137..8a432edbd78 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -20,7 +20,7 @@ module Gitlab keys << cache_key(key) redis.pipelined do - keys.each_slice(1000) { |subset| redis.del(*subset) } + keys.each_slice(1000) { |subset| redis.unlink(*subset) } end end end diff --git a/lib/gitlab/redis/hll.rb b/lib/gitlab/redis/hll.rb new file mode 100644 index 00000000000..ff5754675e2 --- /dev/null +++ b/lib/gitlab/redis/hll.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class HLL + KEY_REGEX = %r{\A(\w|-|:)*\{\w*\}(\w|-|:)*\z}.freeze + KeyFormatError = Class.new(StandardError) + + def self.count(params) + self.new.count(params) + end + + def self.add(params) + self.new.add(params) + end + + def count(keys:) + Gitlab::Redis::SharedState.with do |redis| + redis.pfcount(*keys) + end + end + + # Check a basic format for the Redis key in order to ensure the keys are in the same hash slot + # + # Examples of keys + # project:{1}:set_a + # project:{1}:set_b + # project:{2}:set_c + # 2020-216-{project_action} + # i_{analytics}_dev_ops_score-2020-32 + def add(key:, value:, expiry:) + unless KEY_REGEX.match?(key) + raise KeyFormatError.new("Invalid key format. #{key} key should have changeable parts in curly braces. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands") + end + + Gitlab::Redis::SharedState.with do |redis| + redis.multi do |multi| + multi.pfadd(key, value) + multi.expire(key, expiry) + end + end + end + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 784f8b48f3c..1e1e0d856b7 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -3,7 +3,7 @@ module Gitlab module Regex module Packages - CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt].freeze + CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze def conan_file_name_regex @@ -160,6 +160,15 @@ module Gitlab "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', '*' and spaces" end + # https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/identity_and_auth.md#agent-identity-and-name + def cluster_agent_name_regex + /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/ + end + + def cluster_agent_name_regex_message + %q{can contain only lowercase letters, digits, and '-', but cannot start or end with '-'} + end + def kubernetes_namespace_regex /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/ end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 688a4a39dba..f6a5c6ed754 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -58,11 +58,19 @@ module Gitlab # wrong answer. We handle that by querying the full list - which fills # the cache - and using it directly to answer the question. define_method("#{name}_include?") do |value| - if strong_memoized?(name) || !redis_set_cache.exist?(name) - return __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend - end + ivar = "@#{name}_include" + memoized = instance_variable_get(ivar) || {} + + next memoized[value] if memoized.key?(value) + + memoized[value] = + if strong_memoized?(name) || !redis_set_cache.exist?(name) + __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend + else + redis_set_cache.include?(name, value) + end - redis_set_cache.include?(name, value) + instance_variable_set(ivar, memoized)[value] end end @@ -241,7 +249,7 @@ module Gitlab end def expire_redis_hash_method_caches(methods) - methods.each { |name| redis_hash_cache.delete(name) } + redis_hash_cache.delete(*methods) end # All cached repository methods depend on the existence of a Git repository, diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb index d2a7b450000..d479d3115a6 100644 --- a/lib/gitlab/repository_hash_cache.rb +++ b/lib/gitlab/repository_hash_cache.rb @@ -31,10 +31,18 @@ module Gitlab "#{type}:#{namespace}:hash" end - # @param key [String] - # @return [Integer] 0 or 1 depending on success - def delete(key) - with { |redis| redis.del(cache_key(key)) } + # @param keys [String] one or multiple keys to delete + # @return [Integer] the number of keys successfully deleted + def delete(*keys) + return 0 if keys.empty? + + with do |redis| + keys = keys.map { |key| cache_key(key) } + + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.unlink(*keys) + end + end end # Check if the provided hash key exists in the hash. diff --git a/lib/gitlab/search/parsed_query.rb b/lib/gitlab/search/parsed_query.rb index 1f6e0519b4c..5d5d407c172 100644 --- a/lib/gitlab/search/parsed_query.rb +++ b/lib/gitlab/search/parsed_query.rb @@ -3,6 +3,8 @@ module Gitlab module Search class ParsedQuery + include Gitlab::Utils::StrongMemoize + attr_reader :term, :filters def initialize(term, filters) @@ -11,13 +13,44 @@ module Gitlab end def filter_results(results) - filters = @filters.reject { |filter| filter[:matcher].nil? } - return unless filters + with_matcher = ->(filter) { filter[:matcher].present? } + + excluding = excluding_filters.select(&with_matcher) + including = including_filters.select(&with_matcher) + + return unless excluding.any? || including.any? + + results.select! do |result| + including.all? { |filter| filter[:matcher].call(filter, result) } + end + + results.reject! do |result| + excluding.any? { |filter| filter[:matcher].call(filter, result) } + end + + results + end + + private + + def including_filters + processed_filters(:including) + end + + def excluding_filters + processed_filters(:excluding) + end + + def processed_filters(type) + excluding, including = strong_memoize(:processed_filters) do + filters.partition { |filter| filter[:negated] } + end - results.select do |result| - filters.all? do |filter| - filter[:matcher].call(filter, result) - end + case type + when :including then including + when :excluding then excluding + else + raise ArgumentError.new(type) end end end diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb index ba0e16607a6..27ea0b7367f 100644 --- a/lib/gitlab/search/query.rb +++ b/lib/gitlab/search/query.rb @@ -20,7 +20,10 @@ module Gitlab private def filter(name, **attributes) - filter = { parser: @filter_options[:default_parser], name: name }.merge(attributes) + filter = { + parser: @filter_options[:default_parser], + name: name + }.merge(attributes) @filters << filter end @@ -33,12 +36,13 @@ module Gitlab fragments = [] filters = @filters.each_with_object([]) do |filter, parsed_filters| - match = @raw_query.split.find { |part| part =~ /\A#{filter[:name]}:/ } + match = @raw_query.split.find { |part| part =~ /\A-?#{filter[:name]}:/ } next unless match input = match.split(':')[1..-1].join next if input.empty? + filter[:negated] = match.start_with?("-") filter[:value] = parse_filter(filter, input) filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?') fragments << match diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index d652719721e..e26d45e1b33 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -66,7 +66,7 @@ module Gitlab estimated_minutes = (size.to_f / ESTIMATED_INSERT_PER_MINUTE).round humanized_minutes = 'minute'.pluralize(estimated_minutes) - if estimated_minutes.zero? + if estimated_minutes == 0 "Rough estimated time: less than a minute ⏰" else "Rough estimated time: #{estimated_minutes} #{humanized_minutes} ⏰" diff --git a/lib/gitlab/service_desk_email.rb b/lib/gitlab/service_desk_email.rb index f8dba82cb40..52da10eff3e 100644 --- a/lib/gitlab/service_desk_email.rb +++ b/lib/gitlab/service_desk_email.rb @@ -17,6 +17,12 @@ module Gitlab def config Gitlab.config.service_desk_email end + + def address_for_key(key) + return if config.address.blank? + + config.address.sub(Gitlab::IncomingEmail::WILDCARD_PLACEHOLDER, key) + end end end end diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 6ba9ee26634..591265d014e 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -22,7 +22,7 @@ module Gitlab keys = keys.map { |key| cache_key(key) } Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - unlink_or_delete(redis, keys) + redis.unlink(*keys) end end end @@ -60,17 +60,5 @@ module Gitlab def with(&blk) Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord end - - def unlink_or_delete(redis, keys) - if Feature.enabled?(:repository_set_cache_unlink, default_enabled: true) - redis.unlink(*keys) - else - redis.del(*keys) - end - rescue ::Redis::CommandError => e - Gitlab::ErrorTracking.log_exception(e) - - redis.del(*keys) - end end end diff --git a/lib/gitlab/sidekiq_cluster.rb b/lib/gitlab/sidekiq_cluster.rb index e74ae8d0f03..d05c717d2fa 100644 --- a/lib/gitlab/sidekiq_cluster.rb +++ b/lib/gitlab/sidekiq_cluster.rb @@ -126,7 +126,7 @@ module Gitlab def self.concurrency(queues, min_concurrency, max_concurrency) concurrency_from_queues = queues.length + 1 - max = max_concurrency.positive? ? max_concurrency : concurrency_from_queues + max = max_concurrency > 0 ? max_concurrency : concurrency_from_queues min = [min_concurrency, max].min concurrency_from_queues.clamp(min, max) diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index 9d0d67a488f..b8a4eedd620 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -239,7 +239,7 @@ module Gitlab memory_growth_kb = get_job_options(job, 'memory_killer_memory_growth_kb', 0).to_i max_memory_growth_kb = get_job_options(job, 'memory_killer_max_memory_growth_kb', DEFAULT_MAX_MEMORY_GROWTH_KB).to_i - return 0 if memory_growth_kb.zero? + return 0 if memory_growth_kb == 0 time_elapsed = [Gitlab::Metrics::System.monotonic_time - job[:started_at], 0].max [memory_growth_kb * time_elapsed, max_memory_growth_kb].min diff --git a/lib/gitlab/sidekiq_logger.rb b/lib/gitlab/sidekiq_logger.rb deleted file mode 100644 index ce82a6f04bb..00000000000 --- a/lib/gitlab/sidekiq_logger.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class SidekiqLogger < Gitlab::Logger - def self.file_name_noext - 'sidekiq' - end - end -end diff --git a/lib/gitlab/sidekiq_logging/exception_handler.rb b/lib/gitlab/sidekiq_logging/exception_handler.rb index a6d6819bf8e..8ae6addc2c6 100644 --- a/lib/gitlab/sidekiq_logging/exception_handler.rb +++ b/lib/gitlab/sidekiq_logging/exception_handler.rb @@ -18,7 +18,7 @@ module Gitlab data.merge!(job_data) if job_data.present? end - data[:error_backtrace] = Gitlab::BacktraceCleaner.clean_backtrace(job_exception.backtrace) if job_exception.backtrace.present? + data[:error_backtrace] = Rails.backtrace_cleaner.clean(job_exception.backtrace) if job_exception.backtrace.present? Sidekiq.logger.warn(data) end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 4eef3fbd12e..5f912818605 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -19,6 +19,7 @@ module Gitlab chain.add ::Labkit::Middleware::Sidekiq::Server chain.add ::Gitlab::SidekiqMiddleware::InstrumentationLogger chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Server + chain.add ::Gitlab::SidekiqVersioning::Middleware chain.add ::Gitlab::SidekiqStatus::ServerMiddleware chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Server chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 49c4fdc3033..0b38c98f710 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -55,7 +55,7 @@ module Gitlab def get_rss output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status.zero? + return 0 unless status == 0 output.to_i end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 6a942a6ce06..0635c07ae4b 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -14,6 +14,10 @@ module Gitlab end def call(worker, job, queue) + # This gives all the sidekiq worker threads a name, so we can recognize them + # in metrics and can use them in the `ThreadsSampler` for setting a label + Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME + labels = create_labels(worker.class, queue) queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 0dafccb3d34..2293e2adee1 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -50,7 +50,7 @@ module Gitlab # # Returns true or false. def self.all_completed?(job_ids) - self.num_running(job_ids).zero? + self.num_running(job_ids) == 0 end # Returns true if the given job is running or enqueued. diff --git a/lib/gitlab/sidekiq_versioning/middleware.rb b/lib/gitlab/sidekiq_versioning/middleware.rb new file mode 100644 index 00000000000..2ffee617376 --- /dev/null +++ b/lib/gitlab/sidekiq_versioning/middleware.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqVersioning + class Middleware + def call(worker, job, queue) + worker.job_version = job['version'] if worker.is_a?(ApplicationWorker) + + yield + end + end + end +end diff --git a/lib/gitlab/sidekiq_versioning/worker.rb b/lib/gitlab/sidekiq_versioning/worker.rb new file mode 100644 index 00000000000..fe9bae6b8a1 --- /dev/null +++ b/lib/gitlab/sidekiq_versioning/worker.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqVersioning + module Worker + extend ActiveSupport::Concern + + included do + version 0 + + attr_writer :job_version + end + + class_methods do + def version(new_version = nil) + if new_version + sidekiq_options version: new_version.to_i + else + get_sidekiq_options['version'] + end + end + end + + # Version is not set if `new.perform` is called directly, + # and in that case we fallback to latest version + def job_version + @job_version ||= self.class.version + end + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index b60f0b78fef..b8affb42372 100644 --- a/lib/gitlab/slash_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -69,7 +69,6 @@ module Gitlab def resource_url url_for( [ - resource.project.namespace.becomes(Namespace), resource.project, resource ] diff --git a/lib/gitlab/slash_commands/presenters/issue_search.rb b/lib/gitlab/slash_commands/presenters/issue_search.rb index fffa082baac..f5b1670b2e9 100644 --- a/lib/gitlab/slash_commands/presenters/issue_search.rb +++ b/lib/gitlab/slash_commands/presenters/issue_search.rb @@ -22,7 +22,7 @@ module Gitlab def attachments resource.map do |issue| - url = "[#{issue.to_reference}](#{url_for([namespace, project, issue])})" + url = "[#{issue.to_reference}](#{url_for([project, issue])})" { color: color(issue), @@ -39,10 +39,6 @@ module Gitlab def project @project ||= resource.first.project end - - def namespace - @namespace ||= project.namespace.becomes(Namespace) - end end end end diff --git a/lib/gitlab/slash_commands/presenters/issue_show.rb b/lib/gitlab/slash_commands/presenters/issue_show.rb index 448381b64ed..e9df015f249 100644 --- a/lib/gitlab/slash_commands/presenters/issue_show.rb +++ b/lib/gitlab/slash_commands/presenters/issue_show.rb @@ -23,14 +23,14 @@ module Gitlab def text message = ["**#{status_text(resource)}**"] - if resource.upvotes.zero? && resource.downvotes.zero? && resource.user_notes_count.zero? + if resource.upvotes == 0 && resource.downvotes == 0 && resource.user_notes_count == 0 return message.join end message << " · " - message << ":+1: #{resource.upvotes} " unless resource.upvotes.zero? - message << ":-1: #{resource.downvotes} " unless resource.downvotes.zero? - message << ":speech_balloon: #{resource.user_notes_count}" unless resource.user_notes_count.zero? + message << ":+1: #{resource.upvotes} " unless resource.upvotes == 0 + message << ":-1: #{resource.downvotes} " unless resource.downvotes == 0 + message << ":speech_balloon: #{resource.user_notes_count}" unless resource.user_notes_count == 0 message.join end diff --git a/lib/gitlab/static_site_editor/config.rb b/lib/gitlab/static_site_editor/config.rb index 08ed6599a6e..d335a434335 100644 --- a/lib/gitlab/static_site_editor/config.rb +++ b/lib/gitlab/static_site_editor/config.rb @@ -3,7 +3,7 @@ module Gitlab module StaticSiteEditor class Config - SUPPORTED_EXTENSIONS = %w[.md .md.erb].freeze + SUPPORTED_EXTENSIONS = %w[.md].freeze def initialize(repository, ref, file_path, return_url) @repository = repository @@ -42,6 +42,8 @@ module Gitlab end def extension_supported? + return true if file_path.end_with?('.md.erb') && Feature.enabled?(:sse_erb_support, project) + SUPPORTED_EXTENSIONS.any? { |ext| file_path.end_with?(ext) } end diff --git a/lib/gitlab/suggestions/suggestion_set.rb b/lib/gitlab/suggestions/suggestion_set.rb index abb05ba56a7..f9a635734a3 100644 --- a/lib/gitlab/suggestions/suggestion_set.rb +++ b/lib/gitlab/suggestions/suggestion_set.rb @@ -83,7 +83,7 @@ module Gitlab end unless suggestion.appliable?(cached: false) - return _('A suggestion is not applicable.') + return suggestion.inapplicable_reason(cached: false) end unless latest_source_head?(suggestion) diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 6ccb442b1e0..73187d8dea8 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -95,7 +95,7 @@ module Gitlab def run_command!(command) output, status = Gitlab::Popen.popen(command) - raise Gitlab::TaskFailedError.new(output) unless status.zero? + raise Gitlab::TaskFailedError.new(output) unless status == 0 output end diff --git a/lib/gitlab/template/metrics_dashboard_template.rb b/lib/gitlab/template/metrics_dashboard_template.rb new file mode 100644 index 00000000000..88fc3007b63 --- /dev/null +++ b/lib/gitlab/template/metrics_dashboard_template.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Template + class MetricsDashboardTemplate < BaseTemplate + def content + explanation = "# This file is a template, and might need editing before it works on your project." + [explanation, super].join("\n") + end + + class << self + def extension + '.metrics-dashboard.yml' + end + + def categories + { + "General" => '' + } + end + + def base_dir + Rails.root.join('lib/gitlab/metrics/templates') + end + + def finder(project = nil) + Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) + end + end + end + end +end diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index c237f4a7404..6a3e2062070 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -31,7 +31,7 @@ module Gitlab def scan(text) matches = scan_regexp.scan(text).to_a - matches.map!(&:first) if regexp.number_of_capturing_groups.zero? + matches.map!(&:first) if regexp.number_of_capturing_groups == 0 matches end @@ -68,7 +68,7 @@ module Gitlab # groups, so work around it def scan_regexp @scan_regexp ||= - if regexp.number_of_capturing_groups.zero? + if regexp.number_of_capturing_groups == 0 RE2::Regexp.new('(' + regexp.source + ')') else regexp diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 9d7e6536608..70efe86143e 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -10,11 +10,8 @@ # alt_usage_data { Gitlab::VERSION } # redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) # redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } - module Gitlab class UsageData - BATCH_SIZE = 100 - class << self include Gitlab::Utils::UsageData include Gitlab::Utils::StrongMemoize @@ -40,6 +37,7 @@ module Gitlab .merge(usage_activity_by_stage) .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, last_28_days_time_period)) .merge(analytics_unique_visits_data) + .merge(compliance_unique_visits_data) end end @@ -60,13 +58,12 @@ module Gitlab end def recorded_at - Time.now + Time.current end # rubocop: disable Metrics/AbcSize # rubocop: disable CodeReuse/ActiveRecord def system_usage_data - alert_bot_incident_count = count(::Issue.authored(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id) issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id) { @@ -84,9 +81,11 @@ module Gitlab auto_devops_enabled: count(::ProjectAutoDevops.enabled), auto_devops_disabled: count(::ProjectAutoDevops.disabled), deploy_keys: count(DeployKey), + # rubocop: disable UsageData/LargeTable: deployments: deployment_count(Deployment), successful_deployments: deployment_count(Deployment.success), failed_deployments: deployment_count(Deployment.failed), + # rubocop: enable UsageData/LargeTable: environments: count(::Environment), clusters: count(::Clusters::Cluster), clusters_enabled: count(::Clusters::Cluster.enabled), @@ -122,8 +121,8 @@ module Gitlab issues_created_from_alerts: total_alert_issues, issues_created_gitlab_alerts: issues_created_manually_from_alerts, issues_created_manually_from_alerts: issues_created_manually_from_alerts, - incident_issues: alert_bot_incident_count, - alert_bot_incident_issues: alert_bot_incident_count, + incident_issues: count(::Issue.incident, start: issue_minimum_id, finish: issue_maximum_id), + alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id), incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: issue_minimum_id, finish: issue_maximum_id), keys: count(Key), label_lists: count(List.label), @@ -141,6 +140,7 @@ module Gitlab projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.terraform_reports, :project_id), projects_with_terraform_states: distinct_count(::Terraform::State, :project_id), protected_branches: count(ProtectedBranch), + protected_branches_except_default: count(ProtectedBranch.where.not(name: ['main', 'master', Gitlab::CurrentSettings.default_branch_name])), releases: count(Release), remote_mirrors: count(RemoteMirror), personal_snippets: count(PersonalSnippet), @@ -159,7 +159,8 @@ module Gitlab usage_counters, user_preferences_usage, ingress_modsecurity_usage, - container_expiration_policies_usage + container_expiration_policies_usage, + service_desk_counts ).tap do |data| data[:snippets] = data[:personal_snippets] + data[:project_snippets] end @@ -170,9 +171,11 @@ module Gitlab def system_usage_data_monthly { counts_monthly: { + # rubocop: disable UsageData/LargeTable: deployments: deployment_count(Deployment.where(last_28_days_time_period)), successful_deployments: deployment_count(Deployment.success.where(last_28_days_time_period)), failed_deployments: deployment_count(Deployment.failed.where(last_28_days_time_period)), + # rubocop: enable UsageData/LargeTable: personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)), project_snippets: count(ProjectSnippet.where(last_28_days_time_period)) }.tap do |data| @@ -254,22 +257,17 @@ module Gitlab enabled: alt_usage_data(fallback: nil) { Gitlab.config.pages.enabled }, version: alt_usage_data { Gitlab::Pages::VERSION } }, + container_registry_server: { + vendor: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_vendor }, + version: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_version } + }, database: { adapter: alt_usage_data { Gitlab::Database.adapter_name }, version: alt_usage_data { Gitlab::Database.version } - }, - app_server: { type: app_server_type } + } } end - def app_server_type - Gitlab::Runtime.identify.to_s - rescue Gitlab::Runtime::IdentificationError => e - Gitlab::AppLogger.error(e.message) - Gitlab::ErrorTracking.track_exception(e) - 'unknown_app_server_type' - end - def object_store_config(component) config = alt_usage_data(fallback: nil) do Settings[component]['object_store'] @@ -308,6 +306,7 @@ module Gitlab Gitlab::UsageData::Topology.new.topology_usage_data end + # rubocop: disable UsageData/DistinctCountByLargeForeignKey def ingress_modsecurity_usage ## # This method measures usage of the Modsecurity Web Application Firewall across the entire @@ -328,6 +327,7 @@ module Gitlab ingress_modsecurity_not_installed: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_not_installed), column) } end + # rubocop: enable UsageData/DistinctCountByLargeForeignKey # rubocop: disable CodeReuse/ActiveRecord def container_expiration_policies_usage @@ -336,40 +336,46 @@ module Gitlab finish = ::Project.maximum(:id) results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish) + # rubocop: disable UsageData/LargeTable base = ::ContainerExpirationPolicy.active + # rubocop: enable UsageData/LargeTable results[:projects_with_expiration_policy_enabled] = distinct_count(base, :project_id, start: start, finish: finish) + # rubocop: disable UsageData/LargeTable %i[keep_n cadence older_than].each do |option| ::ContainerExpirationPolicy.public_send("#{option}_options").keys.each do |value| # rubocop: disable GitlabSecurity/PublicSend results["projects_with_expiration_policy_enabled_with_#{option}_set_to_#{value}".to_sym] = distinct_count(base.where(option => value), :project_id, start: start, finish: finish) end end + # rubocop: enable UsageData/LargeTable results[:projects_with_expiration_policy_enabled_with_keep_n_unset] = distinct_count(base.where(keep_n: nil), :project_id, start: start, finish: finish) results[:projects_with_expiration_policy_enabled_with_older_than_unset] = distinct_count(base.where(older_than: nil), :project_id, start: start, finish: finish) results end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def services_usage - Service.available_services_names.without('jira').each_with_object({}) do |service_name, response| - response["projects_#{service_name}_active".to_sym] = count(Service.active.where(template: false, type: "#{service_name}_service".camelize)) - end.merge(jira_usage).merge(jira_import_usage) + # rubocop: disable UsageData/LargeTable: + Service.available_services_names.each_with_object({}) do |service_name, response| + response["projects_#{service_name}_active".to_sym] = count(Service.active.where(template: false, instance: false, type: "#{service_name}_service".camelize)) + response["templates_#{service_name}_active".to_sym] = count(Service.active.where(template: true, type: "#{service_name}_service".camelize)) + response["instances_#{service_name}_active".to_sym] = count(Service.active.where(instance: true, type: "#{service_name}_service".camelize)) + response["projects_inheriting_instance_#{service_name}_active".to_sym] = count(Service.active.where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize)) + end.merge(jira_usage, jira_import_usage) + # rubocop: enable UsageData/LargeTable: end def jira_usage # Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999 # so we can just check for subdomains of atlassian.net - results = { projects_jira_server_active: 0, - projects_jira_cloud_active: 0, - projects_jira_active: 0 + projects_jira_cloud_active: 0 } - JiraService.active.includes(:jira_tracker_data).find_in_batches(batch_size: BATCH_SIZE) do |services| + # rubocop: disable UsageData/LargeTable: + JiraService.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| counts = services.group_by do |service| # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 service_url = service.data_fields&.url || (service.properties && service.properties['url']) @@ -378,23 +384,16 @@ module Gitlab results[:projects_jira_server_active] += counts[:server].size if counts[:server] results[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud] - results[:projects_jira_active] += services.size end - + # rubocop: enable UsageData/LargeTable: results rescue ActiveRecord::StatementInvalid - { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK, projects_jira_active: FALLBACK } - end - - def successful_deployments_with_cluster(scope) - scope - .joins(cluster: :deployments) - .merge(Clusters::Cluster.enabled) - .merge(Deployment.success) + { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK } end # rubocop: enable CodeReuse/ActiveRecord def jira_import_usage + # rubocop: disable UsageData/LargeTable finished_jira_imports = JiraImportState.finished { @@ -402,21 +401,28 @@ module Gitlab jira_imports_projects_count: distinct_count(finished_jira_imports, :project_id), jira_imports_total_imported_issues_count: alt_usage_data { JiraImportState.finished_imports_count } } + # rubocop: enable UsageData/LargeTable end + # rubocop: disable CodeReuse/ActiveRecord + # rubocop: disable UsageData/LargeTable + def successful_deployments_with_cluster(scope) + scope + .joins(cluster: :deployments) + .merge(Clusters::Cluster.enabled) + .merge(Deployment.success) + end + # rubocop: enable UsageData/LargeTable + # rubocop: enable CodeReuse/ActiveRecord + def user_preferences_usage {} # augmented in EE end # rubocop: disable CodeReuse/ActiveRecord def merge_requests_users(time_period) - query = - Event - .where(target_type: Event::TARGET_TYPES[:merge_request].to_s) - .where(time_period) - distinct_count( - query, + Event.where(target_type: Event::TARGET_TYPES[:merge_request].to_s).where(time_period), :author_id, start: user_minimum_id, finish: user_maximum_id @@ -454,6 +460,7 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord + # rubocop: disable UsageData/LargeTable def usage_activity_by_stage_configure(time_period) { clusters_applications_cert_managers: cluster_applications_user_distinct_count(::Clusters::Applications::CertManager, time_period), @@ -474,6 +481,7 @@ module Gitlab project_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.project_type, time_period) } end + # rubocop: enable UsageData/LargeTable # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord @@ -532,7 +540,9 @@ module Gitlab issues: distinct_count(::Issue.where(time_period), :author_id), notes: distinct_count(::Note.where(time_period), :author_id), projects: distinct_count(::Project.where(time_period), :creator_id), - todos: distinct_count(::Todo.where(time_period), :author_id) + todos: distinct_count(::Todo.where(time_period), :author_id), + service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period), + service_desk_issues: count(::Issue.service_desk.where(time_period)) } end # rubocop: enable CodeReuse/ActiveRecord @@ -574,21 +584,30 @@ module Gitlab end def analytics_unique_visits_data - results = ::Gitlab::Analytics::UniqueVisits::TARGET_IDS.each_with_object({}) do |target_id, hash| - hash[target_id] = redis_usage_data { unique_visit_service.weekly_unique_visits_for_target(target_id) } + results = ::Gitlab::Analytics::UniqueVisits.analytics_ids.each_with_object({}) do |target_id, hash| + hash[target_id] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target_id) } end - results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.weekly_unique_visits_for_any_target } + results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics) } + results['analytics_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics, start_date: 4.weeks.ago.to_date, end_date: Date.current) } { analytics_unique_visits: results } end - def action_monthly_active_users(time_period) - return {} unless Feature.enabled?(Gitlab::UsageDataCounters::TrackUniqueActions::FEATURE_FLAG) + def compliance_unique_visits_data + results = ::Gitlab::Analytics::UniqueVisits.compliance_ids.each_with_object({}) do |target_id, hash| + hash[target_id] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target_id) } + end + results['compliance_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance) } + results['compliance_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance, start_date: 4.weeks.ago.to_date, end_date: Date.current) } + { compliance_unique_visits: results } + end + + def action_monthly_active_users(time_period) counter = Gitlab::UsageDataCounters::TrackUniqueActions project_count = redis_usage_data do - counter.count_unique_events( + counter.count_unique( event_action: Gitlab::UsageDataCounters::TrackUniqueActions::PUSH_ACTION, date_from: time_period[:created_at].first, date_to: time_period[:created_at].last @@ -596,7 +615,7 @@ module Gitlab end design_count = redis_usage_data do - counter.count_unique_events( + counter.count_unique( event_action: Gitlab::UsageDataCounters::TrackUniqueActions::DESIGN_ACTION, date_from: time_period[:created_at].first, date_to: time_period[:created_at].last @@ -604,7 +623,7 @@ module Gitlab end wiki_count = redis_usage_data do - counter.count_unique_events( + counter.count_unique( event_action: Gitlab::UsageDataCounters::TrackUniqueActions::WIKI_ACTION, date_from: time_period[:created_at].first, date_to: time_period[:created_at].last @@ -620,6 +639,31 @@ module Gitlab private + def distinct_count_service_desk_enabled_projects(time_period) + project_creator_id_start = user_minimum_id + project_creator_id_finish = user_maximum_id + + distinct_count(::Project.service_desk_enabled.where(time_period), :creator_id, start: project_creator_id_start, finish: project_creator_id_finish) # rubocop: disable CodeReuse/ActiveRecord + end + + # rubocop: disable CodeReuse/ActiveRecord + def service_desk_counts + # rubocop: disable UsageData/LargeTable: + projects_with_service_desk = ::Project.where(service_desk_enabled: true) + # rubocop: enable UsageData/LargeTable: + { + service_desk_enabled_projects: count(projects_with_service_desk), + service_desk_issues: count( + ::Issue.where( + project: projects_with_service_desk, + author: ::User.support_bot, + confidential: true + ) + ) + } + end + # rubocop: enable CodeReuse/ActiveRecord + def unique_visit_service strong_memoize(:unique_visit_service) do ::Gitlab::Analytics::UniqueVisits.new @@ -685,9 +729,11 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord + # rubocop: disable UsageData/DistinctCountByLargeForeignKey def cluster_applications_user_distinct_count(applications, time_period) distinct_count(applications.where(time_period).available.joins(:cluster), 'clusters.user_id') end + # rubocop: enable UsageData/DistinctCountByLargeForeignKey def clusters_user_distinct_count(clusters, time_period) distinct_count(clusters.where(time_period), :user_id) diff --git a/lib/gitlab/usage_data/topology.rb b/lib/gitlab/usage_data/topology.rb index 4bca2cb07e4..edc4dc75750 100644 --- a/lib/gitlab/usage_data/topology.rb +++ b/lib/gitlab/usage_data/topology.rb @@ -17,6 +17,9 @@ module Gitlab 'registry' => 'registry' }.freeze + # If these errors occur, all subsequent queries are likely to fail for the same error + TIMEOUT_ERRORS = [Errno::ETIMEDOUT, Net::OpenTimeout, Net::ReadTimeout].freeze + CollectionFailure = Struct.new(:query, :error) do def to_h { query => error } @@ -51,7 +54,7 @@ module Gitlab def topology_app_requests_per_hour(client) result = query_safely('gitlab_usage_ping:ops:rate5m', 'app_requests', fallback: nil) do |query| - client.query(one_week_average(query)).first + client.query(aggregate_one_week(query)).first end return unless result @@ -63,7 +66,9 @@ module Gitlab def topology_node_data(client) # node-level data by_instance_mem = topology_node_memory(client) + by_instance_mem_utilization = topology_node_memory_utilization(client) by_instance_cpus = topology_node_cpus(client) + by_instance_cpu_utilization = topology_node_cpu_utilization(client) by_instance_uname_info = topology_node_uname_info(client) # service-level data by_instance_by_job_by_type_memory = topology_all_service_memory(client) @@ -73,7 +78,9 @@ module Gitlab @instances.map do |instance| { node_memory_total_bytes: by_instance_mem[instance], + node_memory_utilization: by_instance_mem_utilization[instance], node_cpus: by_instance_cpus[instance], + node_cpu_utilization: by_instance_cpu_utilization[instance], node_uname_info: by_instance_uname_info[instance], node_services: topology_node_services( @@ -84,14 +91,26 @@ module Gitlab end def topology_node_memory(client) - query_safely('gitlab_usage_ping:node_memory_total_bytes:avg', 'node_memory', fallback: {}) do |query| - aggregate_by_instance(client, one_week_average(query)) + query_safely('gitlab_usage_ping:node_memory_total_bytes:max', 'node_memory', fallback: {}) do |query| + aggregate_by_instance(client, aggregate_one_week(query, aggregation: :max)) + end + end + + def topology_node_memory_utilization(client) + query_safely('gitlab_usage_ping:node_memory_utilization:avg', 'node_memory_utilization', fallback: {}) do |query| + aggregate_by_instance(client, aggregate_one_week(query), transform_value: :to_f) end end def topology_node_cpus(client) query_safely('gitlab_usage_ping:node_cpus:count', 'node_cpus', fallback: {}) do |query| - aggregate_by_instance(client, one_week_average(query)) + aggregate_by_instance(client, aggregate_one_week(query, aggregation: :max)) + end + end + + def topology_node_cpu_utilization(client) + query_safely('gitlab_usage_ping:node_cpu_utilization:avg', 'node_cpu_utilization', fallback: {}) do |query| + aggregate_by_instance(client, aggregate_one_week(query), transform_value: :to_f) end end @@ -114,25 +133,25 @@ module Gitlab def topology_service_memory_rss(client) query_safely( 'gitlab_usage_ping:node_service_process_resident_memory_bytes:avg', 'service_rss', fallback: {} - ) { |query| aggregate_by_labels(client, one_week_average(query)) } + ) { |query| aggregate_by_labels(client, aggregate_one_week(query)) } end def topology_service_memory_uss(client) query_safely( 'gitlab_usage_ping:node_service_process_unique_memory_bytes:avg', 'service_uss', fallback: {} - ) { |query| aggregate_by_labels(client, one_week_average(query)) } + ) { |query| aggregate_by_labels(client, aggregate_one_week(query)) } end def topology_service_memory_pss(client) query_safely( 'gitlab_usage_ping:node_service_process_proportional_memory_bytes:avg', 'service_pss', fallback: {} - ) { |query| aggregate_by_labels(client, one_week_average(query)) } + ) { |query| aggregate_by_labels(client, aggregate_one_week(query)) } end def topology_all_service_process_count(client) query_safely( 'gitlab_usage_ping:node_service_process:count', 'service_process_count', fallback: {} - ) { |query| aggregate_by_labels(client, one_week_average(query)) } + ) { |query| aggregate_by_labels(client, aggregate_one_week(query)) } end def topology_all_service_server_types(client) @@ -142,6 +161,11 @@ module Gitlab end def query_safely(query, query_name, fallback:) + if timeout_error_exists? + @failures << CollectionFailure.new(query_name, 'timeout_cancellation') + return fallback + end + result = yield query return result if result.present? @@ -153,6 +177,14 @@ module Gitlab fallback end + def timeout_error_exists? + timeout_error_names = TIMEOUT_ERRORS.map(&:to_s).to_set + + @failures.any? do |failure| + timeout_error_names.include?(failure.error) + end + end + def topology_node_services(instance, all_process_counts, all_process_memory, all_server_types) # returns all node service data grouped by service name as the key instance_service_data = @@ -160,14 +192,17 @@ module Gitlab .deep_merge(topology_instance_service_memory(instance, all_process_memory)) .deep_merge(topology_instance_service_server_types(instance, all_server_types)) - # map to list of hashes where service names become values instead, and remove + # map to list of hashes where service names become values instead, and skip # unknown services, since they might not be ours instance_service_data.each_with_object([]) do |entry, list| service, service_metrics = entry - gitlab_service = JOB_TO_SERVICE_NAME[service.to_s] - next unless gitlab_service + service_name = service.to_s.strip - list << { name: gitlab_service }.merge(service_metrics) + if gitlab_service = JOB_TO_SERVICE_NAME[service_name] + list << { name: gitlab_service }.merge(service_metrics) + else + @failures << CollectionFailure.new('service_unknown', service_name) + end end end @@ -210,7 +245,7 @@ module Gitlab def normalize_localhost_address(instance) ip_addr = IPAddr.new(instance) - is_local_ip = ip_addr.loopback? || ip_addr.to_i.zero? + is_local_ip = ip_addr.loopback? || ip_addr.to_i == 0 is_local_ip ? 'localhost' : instance rescue IPAddr::InvalidAddressError @@ -228,17 +263,17 @@ module Gitlab end end - def one_week_average(query) - "avg_over_time (#{query}[1w])" + def aggregate_one_week(query, aggregation: :avg) + "#{aggregation}_over_time (#{query}[1w])" end - def aggregate_by_instance(client, query) - client.aggregate(query) { |metric| normalize_and_track_instance(metric['instance']) } + def aggregate_by_instance(client, query, transform_value: :to_i) + client.aggregate(query, transform_value: transform_value) { |metric| normalize_and_track_instance(metric['instance']) } end # Will retain a composite key that values are mapped to - def aggregate_by_labels(client, query) - client.aggregate(query) do |metric| + def aggregate_by_labels(client, query, transform_value: :to_i) + client.aggregate(query, transform_value: transform_value) do |metric| metric['instance'] = normalize_and_track_instance(metric['instance']) metric end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb new file mode 100644 index 00000000000..c9c39225068 --- /dev/null +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module HLLRedisCounter + DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks + DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days + DEFAULT_REDIS_SLOT = ''.freeze + + UnknownEvent = Class.new(StandardError) + UnknownAggregation = Class.new(StandardError) + + KNOWN_EVENTS_PATH = 'lib/gitlab/usage_data_counters/known_events.yml'.freeze + ALLOWED_AGGREGATIONS = %i(daily weekly).freeze + + # Track event on entity_id + # Increment a Redis HLL counter for unique event_name and entity_id + # + # All events should be added to know_events file lib/gitlab/usage_data_counters/known_events.yml + # + # Event example: + # + # - name: g_compliance_dashboard # Unique event name + # redis_slot: compliance # Optional slot name, if not defined it will use name as a slot, used for totals + # category: compliance # Group events in categories + # expiry: 29 # Optional expiration time in days, default value 29 days for daily and 6.weeks for weekly + # aggregation: daily # Aggregation level, keys are stored daily or weekly + # + # Usage: + # + # * Track event: Gitlab::UsageDataCounters::HLLRedisCounter.track_event(user_id, 'g_compliance_dashboard') + # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current) + class << self + def track_event(entity_id, event_name, time = Time.zone.now) + event = event_for(event_name) + + raise UnknownEvent.new("Unknown event #{event_name}") unless event.present? + + Gitlab::Redis::HLL.add(key: redis_key(event, time), value: entity_id, expiry: expiry(event)) + end + + def unique_events(event_names:, start_date:, end_date:) + events = events_for(Array(event_names)) + + raise 'Events should be in same slot' unless events_in_same_slot?(events) + raise 'Events should be in same category' unless events_in_same_category?(events) + raise 'Events should have same aggregation level' unless events_same_aggregation?(events) + + aggregation = events.first[:aggregation] + + keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date) + + Gitlab::Redis::HLL.count(keys: keys) + end + + def events_for_category(category) + known_events.select { |event| event[:category] == category }.map { |event| event[:name] } + end + + private + + def keys_for_aggregation(aggregation, events:, start_date:, end_date:) + if aggregation.to_sym == :daily + daily_redis_keys(events: events, start_date: start_date, end_date: end_date) + else + weekly_redis_keys(events: events, start_date: start_date, end_date: end_date) + end + end + + def known_events + @known_events ||= YAML.load_file(Rails.root.join(KNOWN_EVENTS_PATH)).map(&:with_indifferent_access) + end + + def known_events_names + known_events.map { |event| event[:name] } + end + + def events_in_same_slot?(events) + slot = events.first[:redis_slot] + events.all? { |event| event[:redis_slot] == slot } + end + + def events_in_same_category?(events) + category = events.first[:category] + events.all? { |event| event[:category] == category } + end + + def events_same_aggregation?(events) + aggregation = events.first[:aggregation] + events.all? { |event| event[:aggregation] == aggregation } + end + + def expiry(event) + return event[:expiry] if event[:expiry].present? + + event[:aggregation].to_sym == :daily ? DEFAULT_DAILY_KEY_EXPIRY_LENGTH : DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH + end + + def event_for(event_name) + known_events.find { |event| event[:name] == event_name } + end + + def events_for(event_names) + known_events.select { |event| event_names.include?(event[:name]) } + end + + def redis_slot(event) + event[:redis_slot] || DEFAULT_REDIS_SLOT + end + + # Compose the key in order to store events daily or weekly + def redis_key(event, time) + raise UnknownEvent.new("Unknown event #{event[:name]}") unless known_events_names.include?(event[:name].to_s) + raise UnknownAggregation.new("Use :daily or :weekly aggregation") unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) + + slot = redis_slot(event) + key = if slot.present? + event[:name].to_s.gsub(slot, "{#{slot}}") + else + "{#{event[:name]}}" + end + + if event[:aggregation].to_sym == :daily + year_day = time.strftime('%G-%j') + "#{year_day}-#{key}" + else + year_week = time.strftime('%G-%V') + "#{key}-#{year_week}" + end + end + + def daily_redis_keys(events:, start_date:, end_date:) + (start_date.to_date..end_date.to_date).map do |date| + events.map { |event| redis_key(event, date) } + end.flatten + end + + def weekly_redis_keys(events:, start_date:, end_date:) + weeks = end_date.to_date.cweek - start_date.to_date.cweek + weeks = 1 if weeks == 0 + + (0..(weeks - 1)).map do |week_increment| + events.map { |event| redis_key(event, start_date + week_increment * 7.days) } + end.flatten + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events.yml new file mode 100644 index 00000000000..b7e516fa8b1 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events.yml @@ -0,0 +1,88 @@ +--- +# Compliance category +- name: g_compliance_dashboard + redis_slot: compliance + category: compliance + expiry: 84 # expiration time in days, equivalent to 12 weeks + aggregation: weekly +- name: g_compliance_audit_events + category: compliance + redis_slot: compliance + expiry: 84 + aggregation: weekly +- name: i_compliance_audit_events + category: compliance + redis_slot: compliance + expiry: 84 + aggregation: weekly +- name: i_compliance_credential_inventory + category: compliance + redis_slot: compliance + expiry: 84 + aggregation: weekly +# Analytics category +- name: g_analytics_contribution + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: g_analytics_insights + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: g_analytics_issues + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: g_analytics_productivity + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: g_analytics_valuestream + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: p_analytics_pipelines + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: p_analytics_code_reviews + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: p_analytics_valuestream + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: p_analytics_insights + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: p_analytics_issues + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: p_analytics_repo + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: i_analytics_cohorts + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: i_analytics_dev_ops_score + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/track_unique_actions.rb b/lib/gitlab/usage_data_counters/track_unique_actions.rb index 9fb5a29748e..0df982572a4 100644 --- a/lib/gitlab/usage_data_counters/track_unique_actions.rb +++ b/lib/gitlab/usage_data_counters/track_unique_actions.rb @@ -4,7 +4,6 @@ module Gitlab module UsageDataCounters module TrackUniqueActions KEY_EXPIRY_LENGTH = 29.days - FEATURE_FLAG = :track_unique_actions WIKI_ACTION = :wiki_action DESIGN_ACTION = :design_action @@ -27,24 +26,22 @@ module Gitlab }).freeze class << self - def track_action(event_action:, event_target:, author_id:, time: Time.zone.now) + def track_event(event_action:, event_target:, author_id:, time: Time.zone.now) return unless Gitlab::CurrentSettings.usage_ping_enabled - return unless Feature.enabled?(FEATURE_FLAG) return unless valid_target?(event_target) return unless valid_action?(event_action) transformed_target = transform_target(event_target) transformed_action = transform_action(event_action, transformed_target) + target_key = key(transformed_action, time) - add_event(transformed_action, author_id, time) + Gitlab::Redis::HLL.add(key: target_key, value: author_id, expiry: KEY_EXPIRY_LENGTH) end - def count_unique_events(event_action:, date_from:, date_to:) + def count_unique(event_action:, date_from:, date_to:) keys = (date_from.to_date..date_to.to_date).map { |date| key(event_action, date) } - Gitlab::Redis::SharedState.with do |redis| - redis.pfcount(*keys) - end + Gitlab::Redis::HLL.count(keys: keys) end private @@ -69,17 +66,6 @@ module Gitlab year_day = date.strftime('%G-%j') "#{year_day}-{#{event_action}}" end - - def add_event(event_action, author_id, date) - target_key = key(event_action, date) - - Gitlab::Redis::SharedState.with do |redis| - redis.multi do |multi| - multi.pfadd(target_key, author_id) - multi.expire(target_key, KEY_EXPIRY_LENGTH) - end - end - end end end end diff --git a/lib/gitlab/usage_data_counters/wiki_page_counter.rb b/lib/gitlab/usage_data_counters/wiki_page_counter.rb index 9cfe0be5bab..6c3fe842344 100644 --- a/lib/gitlab/usage_data_counters/wiki_page_counter.rb +++ b/lib/gitlab/usage_data_counters/wiki_page_counter.rb @@ -2,7 +2,7 @@ module Gitlab::UsageDataCounters class WikiPageCounter < BaseCounter - KNOWN_EVENTS = %w[create update delete].freeze + KNOWN_EVENTS = %w[view create update delete].freeze PREFIX = 'wiki_pages' end end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 1551548d9b4..1c6ddc2e70f 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -5,15 +5,16 @@ module Gitlab extend Gitlab::Cache::RequestCache request_cache_key do - [user&.id, project&.id] + [user&.id, container&.to_global_id] end - attr_reader :user - attr_accessor :project + attr_reader :user, :push_ability + attr_accessor :container - def initialize(user, project: nil) + def initialize(user, container: nil, push_ability: :push_code) @user = user - @project = project + @container = container + @push_ability = push_ability end def can_do_action?(action) @@ -21,7 +22,7 @@ module Gitlab permission_cache[action] = permission_cache.fetch(action) do - user.can?(action, project) + user.can?(action, container) end end @@ -42,20 +43,20 @@ module Gitlab request_cache def can_create_tag?(ref) return false unless can_access_git? - if protected?(ProtectedTag, project, ref) + if protected?(ProtectedTag, ref) protected_tag_accessible_to?(ref, action: :create) else - user.can?(:admin_tag, project) + user.can?(:admin_tag, container) end end request_cache def can_delete_branch?(ref) return false unless can_access_git? - if protected?(ProtectedBranch, project, ref) - user.can?(:push_to_delete_protected_branch, project) + if protected?(ProtectedBranch, ref) + user.can?(:push_to_delete_protected_branch, container) else - user.can?(:push_code, project) + can_push? end end @@ -64,36 +65,36 @@ module Gitlab end request_cache def can_push_to_branch?(ref) - return false unless can_access_git? - return false unless project - - # Checking for an internal project to prevent an infinite loop: - # https://gitlab.com/gitlab-org/gitlab/issues/36805 - if project.internal? - return false unless user.can?(:push_code, project) - else - return false if !user.can?(:push_code, project) && !project.branch_allows_collaboration?(user, ref) - end + return false unless can_access_git? && container && can_collaborate?(ref) + return true unless protected?(ProtectedBranch, ref) - if protected?(ProtectedBranch, project, ref) - protected_branch_accessible_to?(ref, action: :push) - else - true - end + protected_branch_accessible_to?(ref, action: :push) end request_cache def can_merge_to_branch?(ref) return false unless can_access_git? - if protected?(ProtectedBranch, project, ref) + if protected?(ProtectedBranch, ref) protected_branch_accessible_to?(ref, action: :merge) else - user.can?(:push_code, project) + can_push? end end private + def can_push? + user.can?(push_ability, container) + end + + def can_collaborate?(ref) + assert_project! + + # Checking for an internal project or group to prevent an infinite loop: + # https://gitlab.com/gitlab-org/gitlab/issues/36805 + can_push? || (!project.internal? && project.branch_allows_collaboration?(user, ref)) + end + def permission_cache @permission_cache ||= {} end @@ -103,6 +104,8 @@ module Gitlab end def protected_branch_accessible_to?(ref, action:) + assert_project! + ProtectedBranch.protected_ref_accessible_to?( ref, user, project: project, @@ -111,6 +114,8 @@ module Gitlab end def protected_tag_accessible_to?(ref, action:) + assert_project! + ProtectedTag.protected_ref_accessible_to?( ref, user, project: project, @@ -118,8 +123,22 @@ module Gitlab protected_refs: project.protected_tags) end - request_cache def protected?(kind, project, refs) + request_cache def protected?(kind, refs) + assert_project! + kind.protected?(project, refs) end + + def project + container + end + + # Any method that assumes that it is operating on a project should make this + # explicit by calling `#assert_project!`. + # TODO: remove when we make this class polymorphic enough not to care about projects + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/227635 + def assert_project! + raise "No project! #{project.inspect} is not a Project" unless project.is_a?(::Project) + end end end diff --git a/lib/gitlab/user_access_snippet.rb b/lib/gitlab/user_access_snippet.rb index dcd45f9350d..3d1ec800091 100644 --- a/lib/gitlab/user_access_snippet.rb +++ b/lib/gitlab/user_access_snippet.rb @@ -2,6 +2,7 @@ module Gitlab class UserAccessSnippet < UserAccess + extend ::Gitlab::Utils::Override extend ::Gitlab::Cache::RequestCache # TODO: apply override check https://gitlab.com/gitlab-org/gitlab/issues/205677 @@ -9,11 +10,10 @@ module Gitlab [user&.id, snippet&.id] end - attr_reader :snippet + alias_method :snippet, :container def initialize(user, snippet: nil) - @user = user - @snippet = snippet + super(user, container: snippet) @project = snippet&.project end @@ -43,13 +43,9 @@ module Gitlab def can_push_to_branch?(ref) return true if snippet_migration? - - super - return false unless snippet - return false unless can_do_action?(:update_snippet) - true + can_do_action?(:update_snippet) end def can_merge_to_branch?(ref) @@ -59,5 +55,8 @@ module Gitlab def snippet_migration? user&.migration_bot? && snippet end + + override :project + attr_reader :project end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 8f5c1eda456..e2d93e7cd29 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -7,30 +7,50 @@ module Gitlab # Ensure that the relative path will not traverse outside the base directory # We url decode the path to avoid passing invalid paths forward in url encoded format. - # We are ok to pass some double encoded paths to File.open since they won't resolve. # Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580 # It also checks for ALT_SEPARATOR aka '\' (forward slash) - def check_path_traversal!(path, allowed_absolute: false) - path = CGI.unescape(path) - - if path.start_with?("..#{File::SEPARATOR}", "..#{File::ALT_SEPARATOR}") || - path.include?("#{File::SEPARATOR}..#{File::SEPARATOR}") || - path.end_with?("#{File::SEPARATOR}..") || - (!allowed_absolute && Pathname.new(path).absolute?) + def check_path_traversal!(path) + path = decode_path(path) + path_regex = /(\A(\.{1,2})\z|\A\.\.[\/\\]|[\/\\]\.\.\z|[\/\\]\.\.[\/\\]|\n)/ + if path.match?(path_regex) raise PathTraversalAttackError.new('Invalid path') end path end + def allowlisted?(absolute_path, allowlist) + path = absolute_path.downcase + + allowlist.map(&:downcase).any? do |allowed_path| + path.start_with?(allowed_path) + end + end + + def check_allowed_absolute_path!(path, allowlist) + return unless Pathname.new(path).absolute? + return if allowlisted?(path, allowlist) + + raise StandardError, "path #{path} is not allowed" + end + + def decode_path(encoded_path) + decoded = CGI.unescape(encoded_path) + if decoded != CGI.unescape(decoded) + raise StandardError, "path #{encoded_path} is not allowed" + end + + decoded + end + def force_utf8(str) str.dup.force_encoding(Encoding::UTF_8) end def ensure_utf8_size(str, bytes:) raise ArgumentError, 'Empty string provided!' if str.empty? - raise ArgumentError, 'Negative string size provided!' if bytes.negative? + raise ArgumentError, 'Negative string size provided!' if bytes < 0 truncated = str.each_char.each_with_object(+'') do |char, object| if object.bytesize + char.bytesize > bytes diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 625e1076a54..36046ca14bf 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -93,7 +93,7 @@ module Gitlab end def with_finished_at(key, &block) - yield.merge(key => Time.now) + yield.merge(key => Time.current) end private diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index 5d241b9b4e9..9dc687f7740 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -30,6 +30,18 @@ module Gitlab Gitlab::UrlBuilder.instance end + def is_a?(type) + super || subject.is_a?(type) + end + + def web_url + url_builder.build(subject) + end + + def web_path + url_builder.build(subject, only_path: true) + end + class_methods do def presenter? true diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 6d935bb8828..e3b1cb3d016 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -156,6 +156,18 @@ module Gitlab ] end + def send_scaled_image(location, width) + params = { + 'Location' => location, + 'Width' => width + } + + [ + SEND_DATA_HEADER, + "send-scaled-img:#{encode(params)}" + ] + end + def channel_websocket(channel) details = { 'Channel' => { |