diff options
Diffstat (limited to 'lib/gitlab')
271 files changed, 4777 insertions, 2871 deletions
diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 601f2175cfc..760f1352256 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -44,6 +44,10 @@ module Gitlab current.include?(Labkit::Context.log_key(attribute_name)) end + def self.current_context_attribute(attribute_name) + Labkit::Context.current&.get_attribute(attribute_name) + end + def initialize(**args) unknown_attributes = args.keys - APPLICATION_ATTRIBUTES.map(&:name) raise ArgumentError, "#{unknown_attributes} are not known keys" if unknown_attributes.any? diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 4489fc9f3b2..36f58d43a77 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -156,15 +156,16 @@ module Gitlab underscored_service = matched_login['service'].underscore - if Integration.available_services_names.include?(underscored_service) - # We treat underscored_service as a trusted input because it is included - # in the Integration.available_services_names allowlist. - service = project.public_send("#{underscored_service}_service") # rubocop:disable GitlabSecurity/PublicSend + return unless Integration.available_services_names.include?(underscored_service) - if service && service.activated? && service.valid_token?(password) - Gitlab::Auth::Result.new(nil, project, :ci, build_authentication_abilities) - end - end + # We treat underscored_service as a trusted input because it is included + # in the Integration.available_services_names allowlist. + accessor = Project.integration_association_name(underscored_service) + service = project.public_send(accessor) # rubocop:disable GitlabSecurity/PublicSend + + return unless service && service.activated? && service.valid_token?(password) + + Gitlab::Auth::Result.new(nil, project, :ci, build_authentication_abilities) end def user_with_password_for_git(login, password) @@ -371,7 +372,9 @@ module Gitlab end def find_build_by_token(token) - ::Ci::AuthJobFinder.new(token: token).execute + ::Gitlab::Database::LoadBalancing::Session.current.use_primary do + ::Ci::AuthJobFinder.new(token: token).execute + end end def user_auth_attempt!(user, success:) diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 523452d1074..1c5ded2e8ed 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -208,7 +208,7 @@ module Gitlab def build_new_user(skip_confirmation: true) user_params = user_attributes.merge(skip_confirmation: skip_confirmation) - Users::BuildService.new(nil, user_params).execute(skip_authorization: true) + Users::AuthorizedBuildService.new(nil, user_params).execute end def user_attributes diff --git a/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb b/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb new file mode 100644 index 00000000000..cb9b0e88ef4 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The migration is used to cleanup orphaned lfs_objects_projects in order to + # introduce valid foreign keys to this table + class CleanupOrphanedLfsObjectsProjects + # A model to access lfs_objects_projects table in migrations + class LfsObjectsProject < ActiveRecord::Base + self.table_name = 'lfs_objects_projects' + + include ::EachBatch + + belongs_to :lfs_object + belongs_to :project + end + + # A model to access lfs_objects table in migrations + class LfsObject < ActiveRecord::Base + self.table_name = 'lfs_objects' + end + + # A model to access projects table in migrations + class Project < ActiveRecord::Base + self.table_name = 'projects' + end + + SUB_BATCH_SIZE = 5000 + CLEAR_CACHE_DELAY = 1.minute + + def perform(start_id, end_id) + cleanup_lfs_objects_projects_without_lfs_object(start_id, end_id) + cleanup_lfs_objects_projects_without_project(start_id, end_id) + end + + private + + def cleanup_lfs_objects_projects_without_lfs_object(start_id, end_id) + each_record_without_association(start_id, end_id, :lfs_object, :lfs_objects) do |lfs_objects_projects_without_lfs_objects| + projects = Project.where(id: lfs_objects_projects_without_lfs_objects.select(:project_id)) + + if projects.present? + ProjectCacheWorker.bulk_perform_in_with_contexts( + CLEAR_CACHE_DELAY, + projects, + arguments_proc: ->(project) { [project.id, [], [:lfs_objects_size]] }, + context_proc: ->(project) { { project: project } } + ) + end + + lfs_objects_projects_without_lfs_objects.delete_all + end + end + + def cleanup_lfs_objects_projects_without_project(start_id, end_id) + each_record_without_association(start_id, end_id, :project, :projects) do |lfs_objects_projects_without_projects| + lfs_objects_projects_without_projects.delete_all + end + end + + def each_record_without_association(start_id, end_id, association, table_name) + batch = LfsObjectsProject.where(id: start_id..end_id) + + batch.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(lfs_objects_projects.id), max(lfs_objects_projects.id)')).first + + lfs_objects_without_association = + LfsObjectsProject + .unscoped + .left_outer_joins(association) + .where(id: (first..last), table_name => { id: nil }) + + yield lfs_objects_without_association + end + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb b/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb new file mode 100644 index 00000000000..9a88eb8ea06 --- /dev/null +++ b/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + BATCH_SIZE = 1000 + + # This background migration disables container expiration policies connected + # to a project that has no container repositories + class DisableExpirationPoliciesLinkedToNoContainerImages + # rubocop: disable Style/Documentation + class ContainerExpirationPolicy < ActiveRecord::Base + include EachBatch + + self.table_name = 'container_expiration_policies' + end + # rubocop: enable Style/Documentation + + def perform(from_id, to_id) + ContainerExpirationPolicy.where(enabled: true, project_id: from_id..to_id).each_batch(of: BATCH_SIZE) do |batch| + sql = <<-SQL + WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:project_id).limit(BATCH_SIZE).to_sql}) + UPDATE container_expiration_policies + SET enabled = FALSE + FROM batched_relation + WHERE container_expiration_policies.project_id = batched_relation.project_id + AND NOT EXISTS (SELECT 1 FROM "container_repositories" WHERE container_repositories.project_id = container_expiration_policies.project_id) + SQL + execute(sql) + end + end + + private + + def execute(sql) + ActiveRecord::Base + .connection + .execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index 888a12f2330..a00d291245c 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -58,6 +58,13 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid end ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) + + logger.info(message: 'RecalculateVulnerabilitiesOccurrencesUuid Migration: recalculation is done for:', + finding_ids: mappings.keys.pluck(:id)) + + mark_job_as_succeeded(start_id, end_id) + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) end private @@ -76,4 +83,15 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid CalculateFindingUUID.call(name) end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'RecalculateVulnerabilitiesOccurrencesUuid', + arguments + ) + end end diff --git a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb new file mode 100644 index 00000000000..bba1ca26b35 --- /dev/null +++ b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +class Gitlab::BackgroundMigration::UpdateJiraTrackerDataDeploymentTypeBasedOnUrl + # rubocop: disable Gitlab/NamespacedClass + class JiraTrackerData < ActiveRecord::Base + self.table_name = "jira_tracker_data" + self.inheritance_column = :_type_disabled + + include ::Integrations::BaseDataFields + attr_encrypted :url, encryption_options + attr_encrypted :api_url, encryption_options + + enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment + end + # rubocop: enable Gitlab/NamespacedClass + + # https://rubular.com/r/uwgK7k9KH23efa + JIRA_CLOUD_REGEX = %r{^https?://[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?\.atlassian\.net$}ix.freeze + + # rubocop: disable CodeReuse/ActiveRecord + def perform(start_id, end_id) + trackers_data = JiraTrackerData + .where(deployment_type: 'unknown') + .where(id: start_id..end_id) + + cloud, server = trackers_data.partition { |tracker_data| tracker_data.url.match?(JIRA_CLOUD_REGEX) } + + cloud_mappings = cloud.each_with_object({}) do |tracker_data, hash| + hash[tracker_data] = { deployment_type: 2 } + end + + server_mapppings = server.each_with_object({}) do |tracker_data, hash| + hash[tracker_data] = { deployment_type: 1 } + end + + mappings = cloud_mappings.merge(server_mapppings) + + ::Gitlab::Database::BulkUpdate.execute(%i[deployment_type], mappings) + end + # rubocop: enable CodeReuse/ActiveRecord +end diff --git a/lib/gitlab/cache.rb b/lib/gitlab/cache.rb index 90a0c38ff7b..433614a3007 100644 --- a/lib/gitlab/cache.rb +++ b/lib/gitlab/cache.rb @@ -13,6 +13,13 @@ module Gitlab end end end + + # Hook for EE + def delete(key) + Rails.cache.delete(key) + end end end end + +Gitlab::Cache.prepend_mod diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index ec94991157a..86441973941 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -113,6 +113,17 @@ module Gitlab end end + # Returns the values of the given set. + # + # raw_key - The key of the set to check. + def self.values_from_set(raw_key) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.smembers(key) + end + end + # Sets multiple keys to given values. # # mapping - A Hash mapping the cache keys to their values. diff --git a/lib/gitlab/checks/base_bulk_checker.rb b/lib/gitlab/checks/base_bulk_checker.rb new file mode 100644 index 00000000000..46a68fdf485 --- /dev/null +++ b/lib/gitlab/checks/base_bulk_checker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class BaseBulkChecker < BaseChecker + attr_reader :changes_access + delegate(*ChangesAccess::ATTRIBUTES, to: :changes_access) + + def initialize(changes_access) + @changes_access = changes_access + end + + def validate! + raise NotImplementedError + end + end + end +end diff --git a/lib/gitlab/checks/base_checker.rb b/lib/gitlab/checks/base_checker.rb index 68873610408..2b0af7dc4f6 100644 --- a/lib/gitlab/checks/base_checker.rb +++ b/lib/gitlab/checks/base_checker.rb @@ -5,39 +5,16 @@ module Gitlab class BaseChecker include Gitlab::Utils::StrongMemoize - attr_reader :change_access - delegate(*ChangeAccess::ATTRIBUTES, to: :change_access) - - def initialize(change_access) - @change_access = change_access - end - def validate! raise NotImplementedError end private - def creation? - Gitlab::Git.blank_ref?(oldrev) - end - - def deletion? - Gitlab::Git.blank_ref?(newrev) - end - - def update? - !creation? && !deletion? - end - def updated_from_web? protocol == 'web' end - def tag_exists? - project.repository.tag_exists?(tag_name) - end - def validate_once(resource) Gitlab::SafeRequestStore.fetch(cache_key_for_resource(resource)) do yield(resource) diff --git a/lib/gitlab/checks/base_single_checker.rb b/lib/gitlab/checks/base_single_checker.rb new file mode 100644 index 00000000000..f93902055c9 --- /dev/null +++ b/lib/gitlab/checks/base_single_checker.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class BaseSingleChecker < BaseChecker + attr_reader :change_access + delegate(*SingleChangeAccess::ATTRIBUTES, to: :change_access) + + def initialize(change_access) + @change_access = change_access + end + + private + + def creation? + Gitlab::Git.blank_ref?(oldrev) + end + + def deletion? + Gitlab::Git.blank_ref?(newrev) + end + + def update? + !creation? && !deletion? + end + + def tag_exists? + project.repository.tag_exists?(tag_name) + end + end + end +end + +Gitlab::Checks::BaseSingleChecker.prepend_mod_with('Gitlab::Checks::BaseSingleChecker') diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index a8287a97cc3..a2d74d36b58 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class BranchCheck < BaseChecker + class BranchCheck < BaseSingleChecker ERROR_MESSAGES = { delete_default_branch: 'The default branch of a project cannot be deleted.', force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.', diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb new file mode 100644 index 00000000000..4e8b293a3e6 --- /dev/null +++ b/lib/gitlab/checks/changes_access.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class ChangesAccess + ATTRIBUTES = %i[user_access project protocol changes logger].freeze + + attr_reader(*ATTRIBUTES) + + def initialize( + changes, user_access:, project:, protocol:, logger: + ) + @changes = changes + @user_access = user_access + @project = project + @protocol = protocol + @logger = logger + end + + def validate! + return if changes.empty? + + single_access_checks! + + logger.log_timed("Running checks for #{changes.length} changes") do + bulk_access_checks! + end + + true + end + + protected + + def single_access_checks! + # Iterate over all changes to find if user allowed all of them to be applied + changes.each do |change| + # If user does not have access to make at least one change, cancel all + # push by allowing the exception to bubble up + Checks::SingleChangeAccess.new( + change, + user_access: user_access, + project: project, + protocol: protocol, + logger: logger + ).validate! + end + end + + def bulk_access_checks! + Gitlab::Checks::LfsCheck.new(self).validate! + end + end + end +end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index a05181ab58e..d8f5cec8a4a 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class DiffCheck < BaseChecker + class DiffCheck < BaseSingleChecker include Gitlab::Utils::StrongMemoize LOG_MESSAGES = { diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index 38f0b82c8b4..51013b69755 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class LfsCheck < BaseChecker + class LfsCheck < BaseBulkChecker LOG_MESSAGE = 'Scanning repository for blobs stored in LFS and verifying their files have been uploaded to GitLab...' ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' @@ -12,11 +12,10 @@ module Gitlab return unless Feature.enabled?(:lfs_check, default_enabled: true) return unless project.lfs_enabled? - return if skip_lfs_integrity_check - return if deletion? logger.log_timed(LOG_MESSAGE) do - lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) + newrevs = changes.map { |change| change[:newrev] } + lfs_check = Checks::LfsIntegrity.new(project, newrevs, logger.time_left) if lfs_check.objects_missing? raise GitAccess::ForbiddenError, ERROR_MESSAGE diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index 78952db7a3e..845fb2da925 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -3,16 +3,19 @@ module Gitlab module Checks class LfsIntegrity - def initialize(project, newrev, time_left) + def initialize(project, newrevs, time_left) @project = project - @newrev = newrev + @newrevs = newrevs @time_left = time_left end def objects_missing? - return false unless @newrev && @project.lfs_enabled? + return false unless @project.lfs_enabled? - new_lfs_pointers = Gitlab::Git::LfsChanges.new(@project.repository, @newrev) + newrevs = @newrevs.reject { |rev| rev.blank? || Gitlab::Git.blank_ref?(rev) } + return if newrevs.blank? + + new_lfs_pointers = Gitlab::Git::LfsChanges.new(@project.repository, newrevs) .new_pointers(object_limit: ::Gitlab::Git::Repository::REV_LIST_COMMIT_LIMIT, dynamic_timeout: @time_left) return false unless new_lfs_pointers.present? diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb index 2635ad04770..e37cbc0442b 100644 --- a/lib/gitlab/checks/matching_merge_request.rb +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -3,22 +3,74 @@ module Gitlab module Checks class MatchingMergeRequest + TOTAL_METRIC = :gitlab_merge_request_match_total + STALE_METRIC = :gitlab_merge_request_match_stale_secondary + def initialize(newrev, branch_name, project) @newrev = newrev @branch_name = branch_name @project = project end - # rubocop: disable CodeReuse/ActiveRecord def match? + if ::Gitlab::Database::LoadBalancing.enable? + # When a user merges a merge request, the following sequence happens: + # + # 1. Sidekiq: MergeService runs and updates the merge request in a locked state. + # 2. Gitaly: The UserMergeBranch RPC runs. + # 3. Gitaly (gitaly-ruby): This RPC calls the pre-receive hook. + # 4. Rails: This hook makes an API request to /api/v4/internal/allowed. + # 5. Rails: This API check does a SQL query for locked merge + # requests with a matching SHA. + # + # Since steps 1 and 5 will happen on different database + # sessions, replication lag could erroneously cause step 5 to + # report no matching merge requests. To avoid this, we check + # the write location to ensure the replica can make this query. + track_session_metrics do + ::Gitlab::Database::LoadBalancing::Sticking.select_valid_host(:project, @project.id) + end + end + + # rubocop: disable CodeReuse/ActiveRecord @project.merge_requests .with_state(:locked) .where(in_progress_merge_commit_sha: @newrev, target_branch: @branch_name) .exists? + # rubocop: enable CodeReuse/ActiveRecord + end + + private + + def track_session_metrics + before = ::Gitlab::Database::LoadBalancing::Session.current.use_primary? + + yield + + after = ::Gitlab::Database::LoadBalancing::Session.current.use_primary? + + increment_attempt_count + + if !before && after + increment_stale_secondary_count + end + end + + def increment_attempt_count + total_counter.increment + end + + def increment_stale_secondary_count + stale_counter.increment + end + + def total_counter + @total_counter ||= ::Gitlab::Metrics.counter(TOTAL_METRIC, 'Total number of merge request match attempts') + end + + def stale_counter + @stale_counter ||= ::Gitlab::Metrics.counter(STALE_METRIC, 'Total number of merge request match attempts with lagging secondary') end - # rubocop: enable CodeReuse/ActiveRecord end end end - -Gitlab::Checks::MatchingMergeRequest.prepend_mod_with('Gitlab::Checks::MatchingMergeRequest') diff --git a/lib/gitlab/checks/push_check.rb b/lib/gitlab/checks/push_check.rb index 47aa25aae4c..50002e00a77 100644 --- a/lib/gitlab/checks/push_check.rb +++ b/lib/gitlab/checks/push_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class PushCheck < BaseChecker + class PushCheck < BaseSingleChecker def validate! logger.log_timed("Checking if you are allowed to push...") do unless can_push? diff --git a/lib/gitlab/checks/push_file_count_check.rb b/lib/gitlab/checks/push_file_count_check.rb index 288a7e0d41a..707d4cfbcbe 100644 --- a/lib/gitlab/checks/push_file_count_check.rb +++ b/lib/gitlab/checks/push_file_count_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class PushFileCountCheck < BaseChecker + class PushFileCountCheck < BaseSingleChecker attr_reader :repository, :newrev, :limit, :logger LOG_MESSAGES = { diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/single_change_access.rb index a2c3de3e775..280b2dd25e2 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/single_change_access.rb @@ -2,23 +2,22 @@ module Gitlab module Checks - class ChangeAccess + class SingleChangeAccess ATTRIBUTES = %i[user_access project skip_authorization - skip_lfs_integrity_check protocol oldrev newrev ref + protocol oldrev newrev ref branch_name tag_name logger commits].freeze attr_reader(*ATTRIBUTES) def initialize( change, user_access:, project:, - skip_lfs_integrity_check: false, protocol:, logger: + protocol:, logger: ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @tag_name = Gitlab::Git.tag_name(@ref) @user_access = user_access @project = project - @skip_lfs_integrity_check = skip_lfs_integrity_check @protocol = protocol @logger = logger @@ -44,7 +43,6 @@ module Gitlab Gitlab::Checks::PushCheck.new(self).validate! Gitlab::Checks::BranchCheck.new(self).validate! Gitlab::Checks::TagCheck.new(self).validate! - Gitlab::Checks::LfsCheck.new(self).validate! end def commits_check @@ -54,4 +52,4 @@ module Gitlab end end -Gitlab::Checks::ChangeAccess.prepend_mod_with('Gitlab::Checks::ChangeAccess') +Gitlab::Checks::SingleChangeAccess.prepend_mod_with('Gitlab::Checks::SingleChangeAccess') diff --git a/lib/gitlab/checks/snippet_check.rb b/lib/gitlab/checks/snippet_check.rb index d5efbfcc5bc..43168600ec9 100644 --- a/lib/gitlab/checks/snippet_check.rb +++ b/lib/gitlab/checks/snippet_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class SnippetCheck < BaseChecker + class SnippetCheck < BaseSingleChecker ERROR_MESSAGES = { create_delete_branch: 'You can not create or delete branches.' }.freeze diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb index a47e55cb160..a45db85301a 100644 --- a/lib/gitlab/checks/tag_check.rb +++ b/lib/gitlab/checks/tag_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class TagCheck < BaseChecker + class TagCheck < BaseSingleChecker ERROR_MESSAGES = { change_existing_tags: 'You are not allowed to change existing tags on this project.', update_protected_tag: 'Protected tags cannot be updated.', diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index b1dee0e1ecc..466706384c0 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -77,7 +77,7 @@ module Gitlab end def set_section_duration(duration) - @section_duration = Time.at(duration.to_i).strftime('%M:%S') + @section_duration = Time.at(duration.to_i).utc.strftime('%M:%S') end def flush_current_segment! diff --git a/lib/gitlab/ci/badge/coverage/template.rb b/lib/gitlab/ci/badge/coverage/template.rb index 7589fa5ff8b..96702420e9d 100644 --- a/lib/gitlab/ci/badge/coverage/template.rb +++ b/lib/gitlab/ci/badge/coverage/template.rb @@ -24,26 +24,10 @@ module Gitlab::Ci @key_width = badge.customization.dig(:key_width) end - def key_text - if @key_text && @key_text.size <= MAX_KEY_TEXT_SIZE - @key_text - else - @entity.to_s - end - end - def value_text @status ? ("%.2f%%" % @status) : 'unknown' end - def key_width - if @key_width && @key_width.between?(1, MAX_KEY_WIDTH) - @key_width - else - 62 - end - end - def value_width @status ? 54 : 58 end diff --git a/lib/gitlab/ci/badge/pipeline/template.rb b/lib/gitlab/ci/badge/pipeline/template.rb index 8430b01fc9a..c39f96e4a34 100644 --- a/lib/gitlab/ci/badge/pipeline/template.rb +++ b/lib/gitlab/ci/badge/pipeline/template.rb @@ -28,26 +28,10 @@ module Gitlab::Ci @key_width = badge.customization.dig(:key_width) end - def key_text - if @key_text && @key_text.size <= MAX_KEY_TEXT_SIZE - @key_text - else - @entity.to_s - end - end - def value_text STATUS_RENAME[@status.to_s] || @status.to_s end - def key_width - if @key_width && @key_width.between?(1, MAX_KEY_WIDTH) - @key_width - else - 62 - end - end - def value_width 54 end diff --git a/lib/gitlab/ci/badge/template.rb b/lib/gitlab/ci/badge/template.rb index 0580dad72ba..d514a8577bd 100644 --- a/lib/gitlab/ci/badge/template.rb +++ b/lib/gitlab/ci/badge/template.rb @@ -8,6 +8,7 @@ module Gitlab::Ci class Template MAX_KEY_TEXT_SIZE = 64 MAX_KEY_WIDTH = 512 + DEFAULT_KEY_WIDTH = 62 def initialize(badge) @entity = badge.entity @@ -15,7 +16,11 @@ module Gitlab::Ci end def key_text - raise NotImplementedError + if @key_text && @key_text.size <= MAX_KEY_TEXT_SIZE + @key_text + else + @entity.to_s + end end def value_text @@ -23,7 +28,11 @@ module Gitlab::Ci end def key_width - raise NotImplementedError + if @key_width && @key_width.between?(1, MAX_KEY_WIDTH) + @key_width + else + DEFAULT_KEY_WIDTH + end end def value_width diff --git a/lib/gitlab/ci/build/auto_retry.rb b/lib/gitlab/ci/build/auto_retry.rb index e6ef12975c2..b98d1d7b330 100644 --- a/lib/gitlab/ci/build/auto_retry.rb +++ b/lib/gitlab/ci/build/auto_retry.rb @@ -7,6 +7,11 @@ class Gitlab::Ci::Build::AutoRetry scheduler_failure: 2 }.freeze + RETRY_OVERRIDES = { + ci_quota_exceeded: 0, + no_matching_runner: 0 + }.freeze + def initialize(build) @build = build end @@ -19,13 +24,18 @@ class Gitlab::Ci::Build::AutoRetry private + delegate :failure_reason, to: :@build + def within_max_retry_limit? max_allowed_retries > 0 && max_allowed_retries > @build.retries_count end def max_allowed_retries strong_memoize(:max_allowed_retries) do - options_retry_max || DEFAULT_RETRIES.fetch(@build.failure_reason.to_sym, 0) + RETRY_OVERRIDES[failure_reason.to_sym] || + options_retry_max || + DEFAULT_RETRIES[failure_reason.to_sym] || + 0 end end @@ -38,7 +48,7 @@ class Gitlab::Ci::Build::AutoRetry end def retry_on_reason_or_always? - options_retry_when.include?(@build.failure_reason.to_s) || + options_retry_when.include?(failure_reason.to_s) || options_retry_when.include?('always') end diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb index 29dc48c7b42..f1b67635c08 100644 --- a/lib/gitlab/ci/config/entry/need.rb +++ b/lib/gitlab/ci/config/entry/need.rb @@ -35,14 +35,9 @@ module Gitlab end def value - if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) - { name: @config, - artifacts: true, - optional: false } - else - { name: @config, - artifacts: true } - end + { name: @config, + artifacts: true, + optional: false } end end @@ -66,14 +61,9 @@ module Gitlab end def value - if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) - { name: job, - artifacts: artifacts || artifacts.nil?, - optional: !!optional } - else - { name: job, - artifacts: artifacts || artifacts.nil? } - end + { name: job, + artifacts: artifacts || artifacts.nil?, + optional: !!optional } end end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 947b6787aa0..79dfb0eec1d 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -98,7 +98,6 @@ module Gitlab def validate_against_warnings # If rules are valid format and workflow rules are not specified return unless rules_value - return unless Gitlab::Ci::Features.raise_job_rules_without_workflow_rules_warning? last_rule = rules_value.last diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index f2fd8ac7fd9..4db25fb0930 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -13,7 +13,7 @@ module Gitlab ALLOWED_KEYS = %i[junit codequality sast secret_detection dependency_scanning container_scanning - dast performance browser_performance load_performance license_management license_scanning metrics lsif + dast performance browser_performance load_performance license_scanning metrics lsif dotenv cobertura terraform accessibility cluster_applications requirements coverage_fuzzing api_fuzzing].freeze @@ -36,7 +36,6 @@ module Gitlab validates :performance, array_of_strings_or_string: true validates :browser_performance, array_of_strings_or_string: true validates :load_performance, array_of_strings_or_string: true - validates :license_management, array_of_strings_or_string: true validates :license_scanning, array_of_strings_or_string: true validates :metrics, array_of_strings_or_string: true validates :lsif, array_of_strings_or_string: true @@ -44,7 +43,7 @@ module Gitlab validates :cobertura, array_of_strings_or_string: true validates :terraform, array_of_strings_or_string: true validates :accessibility, array_of_strings_or_string: true - validates :cluster_applications, array_of_strings_or_string: true + validates :cluster_applications, array_of_strings_or_string: true # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441 validates :requirements, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index a8f78b62d8d..e6ff33d6f79 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -28,11 +28,6 @@ module Gitlab end end - def matching? - super && - Feature.enabled?(:ci_dynamic_child_pipeline, project, default_enabled: true) - end - private def project diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index c4b4a7a0a73..47441fa3818 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -6,7 +6,7 @@ module Gitlab module External module File class Template < Base - attr_reader :location, :project + attr_reader :location SUFFIX = '.gitlab-ci.yml' @@ -41,7 +41,7 @@ module Gitlab end def fetch_template_content - Gitlab::Template::GitlabCiYmlTemplate.find(template_name, project)&.content + Gitlab::Template::GitlabCiYmlTemplate.find(template_name, context.project)&.content end end end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index efd48a9b29f..bc03658aab8 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -6,6 +6,10 @@ module Gitlab VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC' VALID_SYNTAX_SAMPLE_CRON = '* * * * *' + def self.parse_natural(expression, cron_timezone = 'UTC') + new(Fugit::Nat.parse(expression)&.original, cron_timezone) + end + def initialize(cron, cron_timezone = 'UTC') @cron = cron @cron_timezone = timezone_name(cron_timezone) @@ -27,6 +31,10 @@ module Gitlab try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_timezone).present? end + def match?(time) + cron_line.match?(time) + end + private def timezone_name(timezone) diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index c8e4d9ed763..fe69a170404 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -18,14 +18,6 @@ module Gitlab Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project, default_enabled: true) end - def self.merge_base_pipeline_for_metrics_comparison?(project) - Feature.enabled?(:merge_base_pipeline_for_metrics_comparison, project, default_enabled: :yaml) - end - - def self.raise_job_rules_without_workflow_rules_warning? - ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true) - end - # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` # is a safe switch to disable the feature for a particular project when something went wrong, # therefore it's not supposed to be enabled by default. @@ -33,10 +25,6 @@ module Gitlab ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project) end - def self.trace_overwrite? - ::Feature.enabled?(:ci_trace_overwrite, type: :ops, default_enabled: false) - end - def self.accept_trace?(project) ::Feature.enabled?(:ci_enable_live_trace, project) && ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: true) @@ -53,10 +41,6 @@ module Gitlab def self.gldropdown_tags_enabled? ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) end - - def self.background_pipeline_retry_endpoint?(project) - ::Feature.enabled?(:background_pipeline_retry_endpoint, project) - end end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 0b94debb24e..3fb86b8b3e8 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -54,6 +54,7 @@ module Gitlab user_login: user&.username, user_email: user&.email, pipeline_id: build.pipeline.id.to_s, + pipeline_source: build.pipeline.source.to_s, job_id: build.id.to_s, ref: source_ref, ref_type: ref_type, diff --git a/lib/gitlab/ci/matching/build_matcher.rb b/lib/gitlab/ci/matching/build_matcher.rb new file mode 100644 index 00000000000..dff7d9141d9 --- /dev/null +++ b/lib/gitlab/ci/matching/build_matcher.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Matching + class BuildMatcher + ATTRIBUTES = %i[ + protected + tag_list + build_ids + project + ].freeze + + attr_reader(*ATTRIBUTES) + alias_method :protected?, :protected + + def initialize(params) + ATTRIBUTES.each do |attribute| + instance_variable_set("@#{attribute}", params.fetch(attribute)) + end + end + + def has_tags? + tag_list.present? + end + end + end + end +end diff --git a/lib/gitlab/ci/matching/runner_matcher.rb b/lib/gitlab/ci/matching/runner_matcher.rb new file mode 100644 index 00000000000..63642674936 --- /dev/null +++ b/lib/gitlab/ci/matching/runner_matcher.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Matching + ### + # This class is used to check if a build can be picked by a runner: + # + # runner = Ci::Runner.find(id) + # build = Ci::Build.find(id) + # runner.runner_matcher.matches?(build.build_matcher) + # + # There are also class level methods to build matchers: + # + # `project.builds.build_matchers(project)` returns a distinct collection + # of build matchers. + # `Ci::Runner.runner_matchers` returns a distinct collection of runner matchers. + # + class RunnerMatcher + ATTRIBUTES = %i[ + runner_type + public_projects_minutes_cost_factor + private_projects_minutes_cost_factor + run_untagged + access_level + tag_list + ].freeze + + attr_reader(*ATTRIBUTES) + + def initialize(params) + ATTRIBUTES.each do |attribute| + instance_variable_set("@#{attribute}", params.fetch(attribute)) + end + end + + def matches?(build_matcher) + ensure_build_matcher_instance!(build_matcher) + return false if ref_protected? && !build_matcher.protected? + + accepting_tags?(build_matcher) + end + + def instance_type? + runner_type.to_sym == :instance_type + end + + private + + def ref_protected? + access_level.to_sym == :ref_protected + end + + def accepting_tags?(build_matcher) + (run_untagged || build_matcher.has_tags?) && (build_matcher.tag_list - tag_list).empty? + end + + def ensure_build_matcher_instance!(build_matcher) + return if build_matcher.is_a?(Matching::BuildMatcher) + + raise ArgumentError, 'only Gitlab::Ci::Matching::BuildMatcher are allowed' + end + end + end + end +end + +Gitlab::Ci::Matching::RunnerMatcher.prepend_mod_with('Gitlab::Ci::Matching::RunnerMatcher') diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index ca7fbde6713..364ae66844e 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -69,6 +69,7 @@ module Gitlab elsif data.key?('error') status = ::Gitlab::Ci::Reports::TestCase::STATUS_ERROR system_output = data['error'] + attachment = attachment_path(data['system_out']) elsif data.key?('skipped') status = ::Gitlab::Ci::Reports::TestCase::STATUS_SKIPPED system_output = data['skipped'] diff --git a/lib/gitlab/ci/pipeline/chain/validate/after_config.rb b/lib/gitlab/ci/pipeline/chain/validate/after_config.rb new file mode 100644 index 00000000000..c3db00b4fb2 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/after_config.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class AfterConfig < Chain::Base + include Chain::Helpers + + def perform! + end + + def break? + @pipeline.errors.any? + end + end + end + end + end + end +end + +Gitlab::Ci::Pipeline::Chain::Validate::AfterConfig.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Validate::AfterConfig') diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 539b44513f0..27bb7fdc05a 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -12,12 +12,9 @@ module Gitlab DEFAULT_VALIDATION_REQUEST_TIMEOUT = 5 ACCEPTED_STATUS = 200 - DOT_COM_REJECTED_STATUS = 406 - GENERAL_REJECTED_STATUS = (400..499).freeze + REJECTED_STATUS = 406 def perform! - return unless enabled? - pipeline_authorized = validate_external log_message = pipeline_authorized ? 'authorized' : 'not authorized' @@ -32,24 +29,17 @@ module Gitlab private - def enabled? - return true unless Gitlab.com? - - ::Feature.enabled?(:ci_external_validation_service, project, default_enabled: :yaml) - end - def validate_external return true unless validation_service_url # 200 - accepted - # 406 - not accepted on GitLab.com - # 4XX - not accepted for other installations + # 406 - rejected # everything else - accepted and logged response_code = validate_service_request.code case response_code when ACCEPTED_STATUS true - when rejected_status + when REJECTED_STATUS false else raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}" @@ -60,14 +50,6 @@ module Gitlab true end - def rejected_status - if Gitlab.com? - DOT_COM_REJECTED_STATUS - else - GENERAL_REJECTED_STATUS - end - end - def validate_service_request headers = { 'X-Gitlab-Correlation-id' => Labkit::Correlation::CorrelationId.current_id, @@ -107,7 +89,9 @@ module Gitlab id: current_user.id, username: current_user.username, email: current_user.email, - created_at: current_user.created_at&.iso8601 + created_at: current_user.created_at&.iso8601, + current_sign_in_ip: current_user.current_sign_in_ip, + last_sign_in_ip: current_user.last_sign_in_ip }, pipeline: { sha: pipeline.sha, diff --git a/lib/gitlab/ci/pipeline/preloader.rb b/lib/gitlab/ci/pipeline/preloader.rb index 7befc126ca9..31ddf2c4241 100644 --- a/lib/gitlab/ci/pipeline/preloader.rb +++ b/lib/gitlab/ci/pipeline/preloader.rb @@ -20,6 +20,7 @@ module Gitlab preloader.preload_ref_commits preloader.preload_pipeline_warnings preloader.preload_stages_warnings + preloader.preload_persisted_environments end end end @@ -54,6 +55,13 @@ module Gitlab def preload_stages_warnings @pipeline.stages.each { |stage| stage.number_of_warnings } end + + # This batch loads the associated environments of multiple actions (builds) + # that can't use `preload` due to the indirect relationship. + def preload_persisted_environments + @pipeline.scheduled_actions.each { |action| action.persisted_environment } + @pipeline.manual_actions.each { |action| action.persisted_environment } + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 39dee7750d6..299b27a5f13 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -146,7 +146,7 @@ module Gitlab end @needs_attributes.flat_map do |need| - next if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) && need[:optional] + next if need[:optional] result = @previous_stages.any? do |stage| stage.seeds_names.include?(need[:name]) diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 46e4373ec85..859aeb35f26 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -20,6 +20,8 @@ module Gitlab :build_can_pick, :build_not_pick, :build_not_pending, + :build_queue_push, + :build_queue_pop, :build_temporary_locked, :build_conflict_lock, :build_conflict_exception, @@ -31,7 +33,9 @@ module Gitlab :queue_replication_lag, :runner_pre_assign_checks_failed, :runner_pre_assign_checks_success, - :runner_queue_tick + :runner_queue_tick, + :shared_runner_build_new, + :shared_runner_build_done ].to_set.freeze QUEUE_DEPTH_HISTOGRAMS = [ @@ -77,11 +81,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def increment_queue_operation(operation) - if !Rails.env.production? && !OPERATION_COUNTERS.include?(operation) - raise ArgumentError, "unknown queue operation: #{operation}" - end - - self.class.queue_operations_total.increment(operation: operation) + self.class.increment_queue_operation(operation) end def observe_queue_depth(queue, size) @@ -121,6 +121,14 @@ module Gitlab result end + def self.increment_queue_operation(operation) + if !Rails.env.production? && !OPERATION_COUNTERS.include?(operation) + raise ArgumentError, "unknown queue operation: #{operation}" + end + + queue_operations_total.increment(operation: operation) + end + def self.observe_active_runners(runners_proc) return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb index 239fc3b15e7..287a03cefe2 100644 --- a/lib/gitlab/ci/reports/test_suite_comparer.rb +++ b/lib/gitlab/ci/reports/test_suite_comparer.rb @@ -8,6 +8,7 @@ module Gitlab DEFAULT_MAX_TESTS = 100 DEFAULT_MIN_TESTS = 10 + TestSummary = Struct.new(:new_failures, :existing_failures, :resolved_failures, :new_errors, :existing_errors, :resolved_errors, keyword_init: true) attr_reader :name, :base_suite, :head_suite @@ -90,7 +91,7 @@ module Gitlab def limited_tests strong_memoize(:limited_tests) do # rubocop: disable CodeReuse/ActiveRecord - OpenStruct.new( + TestSummary.new( new_failures: new_failures.take(max_tests), existing_failures: existing_failures.take(max_tests(new_failures)), resolved_failures: resolved_failures.take(max_tests(new_failures, existing_failures)), diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index cbd72f54ff4..66f51f63585 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -30,7 +30,8 @@ module Gitlab reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines', project_deleted: 'pipeline project was deleted', user_blocked: 'pipeline user was blocked', - ci_quota_exceeded: 'no more CI minutes available' + ci_quota_exceeded: 'no more CI minutes available', + no_matching_runner: 'no matching runner available' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index a13f2046291..5680950bba8 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -11,7 +11,7 @@ # * test: TEST_DISABLED # * code_quality: CODE_QUALITY_DISABLED # * license_management: LICENSE_MANAGEMENT_DISABLED -# * performance: PERFORMANCE_DISABLED +# * browser_performance: BROWSER_PERFORMANCE_DISABLED # * load_performance: LOAD_PERFORMANCE_DISABLED # * sast: SAST_DISABLED # * secret_detection: SECRET_DETECTION_DISABLED diff --git a/lib/gitlab/ci/templates/Getting-started.yml b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml index 4dc88418671..07d0de5f9e5 100644 --- a/lib/gitlab/ci/templates/Getting-started.yml +++ b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml @@ -16,7 +16,7 @@ build-job: # This job runs in the build stage, which runs first. stage: build script: - echo "Compiling the code..." - - echo "Compile complete. + - echo "Compile complete." unit-test-job: # This job runs in the test stage. stage: test # It only starts when the job in the build stage completes successfully. diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 01907ef9e2e..56899614cc6 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -1,6 +1,6 @@ # Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html -performance: +browser_performance: stage: performance image: docker:19.03.12 allow_failure: true @@ -72,6 +72,6 @@ performance: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$PERFORMANCE_DISABLED' + - if: '$BROWSER_PERFORMANCE_DISABLED' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml index 5216a46745c..56899614cc6 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -72,6 +72,6 @@ browser_performance: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$PERFORMANCE_DISABLED' + - if: '$BROWSER_PERFORMANCE_DISABLED' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index abcb347b146..cf99d722e4d 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,10 +1,10 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.6.0" + image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v1.0.0' variables: - DOCKER_TLS_CERTDIR: "" + DOCKER_TLS_CERTDIR: '' services: - - name: "docker:20.10.6-dind" + - name: 'docker:20.10.6-dind' command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index b29342216fc..48e877684f6 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.23" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 7ad5a9e2bba..00fcfa64a18 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.7" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 29edada4041..530ab1d0f99 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.7" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0" dependencies: [] review: @@ -91,7 +91,7 @@ canary: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy canary + - auto-deploy deploy canary 50 environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -114,7 +114,6 @@ canary: - auto-deploy create_secret - auto-deploy deploy - auto-deploy delete canary - - auto-deploy delete rollout - auto-deploy persist_environment_url environment: name: production @@ -163,9 +162,7 @@ production_manual: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE - - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) - - auto-deploy delete canary + - auto-deploy deploy canary $ROLLOUT_PERCENTAGE - auto-deploy persist_environment_url environment: name: production diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml new file mode 100644 index 00000000000..6af79728dc8 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -0,0 +1,335 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/ +# +# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-variables + +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + + SAST_EXCLUDED_ANALYZERS: "" + SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" + SCAN_KUBERNETES_MANIFESTS: "false" + +sast: + stage: test + artifacts: + reports: + sast: gl-sast-report.json + rules: + - when: never + variables: + SEARCH_MAX_DEPTH: 4 + script: + - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" + - exit 1 + +.sast-analyzer: + extends: sast + allow_failure: true + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + script: + - /analyzer run + +bandit-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.py' + +brakeman-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.rb' + - '**/Gemfile' + +eslint-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.html' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + +flawfinder-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.c' + - '**/*.cpp' + +kubesec-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ + when: never + - if: $CI_COMMIT_BRANCH && + $SCAN_KUBERNETES_MANIFESTS == 'true' + +gosec-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.go' + +.mobsf-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" + +mobsf-android-sast: + extends: .mobsf-sast + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never + - if: $CI_COMMIT_BRANCH && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.apk' + - '**/AndroidManifest.xml' + +mobsf-ios-sast: + extends: .mobsf-sast + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never + - if: $CI_COMMIT_BRANCH && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.ipa' + - '**/*.xcodeproj/*' + +nodejs-scan-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/package.json' + +phpcs-security-audit-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.php' + +pmd-apex-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.cls' + +security-code-scan-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.csproj' + - '**/*.vbproj' + +semgrep-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.py' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + +sobelow-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - 'mix.exs' + +spotbugs-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ + when: never + - if: $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/AndroidManifest.xml' + when: never + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.groovy' + - '**/*.java' + - '**/*.scala' + - '**/*.kt' diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml new file mode 100644 index 00000000000..d0595491400 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -0,0 +1,36 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection +# +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECRETS_ANALYZER_VERSION: "3" + SECRET_DETECTION_EXCLUDED_PATHS: "" + +.secret-analyzer: + stage: test + image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" + services: [] + allow_failure: true + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + artifacts: + reports: + secret_detection: gl-secret-detection-report.json + +secret_detection: + extends: .secret-analyzer + rules: + - if: $SECRET_DETECTION_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + script: + - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi + - if [[ $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH ]]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi + - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME + - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt + - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt + - /analyzer run + - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 6f30fc2dcd5..ca63e942130 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,21 @@ +################################################################################ +# WARNING +################################################################################ +# +# This template is DEPRECATED and scheduled for removal in GitLab 15.0 +# See https://gitlab.com/gitlab-org/gitlab/-/issues/333610 for more context. +# +# To get started with a Cluster Management Project, we instead recommend +# using the updated project template: +# +# - Documentation: https://docs.gitlab.com/ee/user/clusters/management_project_template.html +# - Source code: https://gitlab.com/gitlab-org/project-templates/cluster-management/ +# +################################################################################ + apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.40.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.43.1" environment: name: production variables: @@ -9,11 +24,9 @@ apply: script: - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml only: - refs: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH artifacts: - reports: - cluster_applications: gl-cluster-applications.json when: on_failure paths: - tiller.log diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 275364afae4..1bdaaeede43 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -1,6 +1,6 @@ # Official language image. Look for the different tagged releases at: # https://hub.docker.com/r/library/ruby/tags/ -image: "ruby:2.5" +image: ruby:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 90fad1550ff..0c4c39cbcd6 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -1,279 +1,33 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables - -stages: - - build - - test - - deploy - - fuzz +# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: + FUZZAPI_VERSION: "1" SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - FUZZAPI_PROFILE: Quick - FUZZAPI_VERSION: "1.6" - FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml - FUZZAPI_TIMEOUT: 30 - FUZZAPI_REPORT: gl-api-fuzzing-report.json - FUZZAPI_REPORT_ASSET_PATH: assets - # - FUZZAPI_D_NETWORK: testing-net - # - # Wait up to 5 minutes for API Fuzzer and target url to become - # available (non 500 response to HTTP(s)) - FUZZAPI_SERVICE_START_TIMEOUT: "300" - # FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} - # - -apifuzzer_fuzz_unlicensed: - stage: fuzz - allow_failure: true - rules: - - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null' - - when: never - script: - - | - echo "Error: Your GitLab project is not licensed for API Fuzzing." - - exit 1 apifuzzer_fuzz: stage: fuzz - image: - name: $FUZZAPI_IMAGE - entrypoint: ["/bin/bash", "-l", "-c"] - variables: - FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://localhost:5000 - FUZZAPI_NEW_REPORT: 1 - FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log - TZ: America/Los_Angeles + image: $FUZZAPI_IMAGE allow_failure: true rules: - - if: $FUZZAPI_D_TARGET_IMAGE - when: never - - if: $FUZZAPI_D_WORKER_IMAGE - when: never - - if: $API_FUZZING_DISABLED - when: never - - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ - script: - # - # Validate options - - | - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ - echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ - echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ - exit 1; \ - fi - # - # Run user provided pre-script - - sh -c "$FUZZAPI_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $FUZZAPI_REPORT_ASSET_PATH - # - # Start API Security background process - - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER & - - APISEC_PID=$! - # - # Start scanning - - worker-entry - # - # Run user provided post-script - - sh -c "$FUZZAPI_POST_SCRIPT" - # - # Shutdown API Security - - kill $APISEC_PID - - wait $APISEC_PID - # - artifacts: - when: always - paths: - - $FUZZAPI_REPORT_ASSET_PATH - - $FUZZAPI_REPORT - - $FUZZAPI_LOG_SCANNER - reports: - api_fuzzing: $FUZZAPI_REPORT - -apifuzzer_fuzz_dnd: - stage: fuzz - image: docker:19.03.12 - variables: - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://apifuzzer:5000 - allow_failure: true - rules: - - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null - when: never - if: $API_FUZZING_DISABLED when: never - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ - services: - - docker:19.03.12-dind + - if: $CI_COMMIT_BRANCH script: - # - # - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - # - - docker network create --driver bridge $FUZZAPI_D_NETWORK - # - # Run user provided pre-script - - sh -c "$FUZZAPI_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $FUZZAPI_REPORT_ASSET_PATH - # - # Start peach testing engine container - - | - docker run -d \ - --name apifuzzer \ - --network $FUZZAPI_D_NETWORK \ - -e Proxy:Port=8000 \ - -e TZ=America/Los_Angeles \ - -e GITLAB_FEATURES \ - -p 80:80 \ - -p 5000:5000 \ - -p 8000:8000 \ - -p 514:514 \ - --restart=no \ - $FUZZAPI_IMAGE \ - dotnet /peach/Peach.Web.dll - # - # Start target container - - | - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \ - docker run -d \ - --name target \ - --network $FUZZAPI_D_NETWORK \ - $FUZZAPI_D_TARGET_ENV \ - $FUZZAPI_D_TARGET_PORTS \ - $FUZZAPI_D_TARGET_VOLUME \ - --restart=no \ - $FUZZAPI_D_TARGET_IMAGE \ - ; fi - # - # Start worker container if provided - - | - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ - echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \ - docker run \ - --name worker \ - --network $FUZZAPI_D_NETWORK \ - -e FUZZAPI_API=http://apifuzzer:5000 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_REPORT_ASSET_PATH \ - -e FUZZAPI_NEW_REPORT=1 \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_POSTMAN_COLLECTION \ - -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ - -e CI_PROJECT_URL \ - -e CI_JOB_ID \ - -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ - $FUZZAPI_D_WORKER_ENV \ - $FUZZAPI_D_WORKER_PORTS \ - $FUZZAPI_D_WORKER_VOLUME \ - --restart=no \ - $FUZZAPI_D_WORKER_IMAGE \ - ; fi - # - # Start API Fuzzing provided worker if no other worker present - - | - if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \ - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ - echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ - echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ - exit 1; \ - fi; \ - docker run \ - --name worker \ - --network $FUZZAPI_D_NETWORK \ - -e TZ=America/Los_Angeles \ - -e FUZZAPI_API=http://apifuzzer:5000 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_REPORT_ASSET_PATH \ - -e FUZZAPI_NEW_REPORT=1 \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_POSTMAN_COLLECTION \ - -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ - -e CI_PROJECT_URL \ - -e CI_JOB_ID \ - -v $CI_PROJECT_DIR:/app \ - -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ - -p 81:80 \ - -p 5001:5000 \ - -p 8001:8000 \ - -p 515:514 \ - --restart=no \ - $FUZZAPI_IMAGE \ - worker-entry \ - ; fi - # - # Propagate exit code from api fuzzing scanner (if any) - - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi - # - # Run user provided post-script - - sh -c "$FUZZAPI_POST_SCRIPT" - # - after_script: - # - # Shutdown all containers - - echo "Stopping all containers" - - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi - - docker stop worker - - docker stop apifuzzer - # - # Save docker logs - - docker logs apifuzzer &> gl-api_fuzzing-logs.log - - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi - - docker logs worker &> gl-api_fuzzing-worker-logs.log - # + - /peach/analyzer-fuzz-api artifacts: when: always paths: - - ./gl-api_fuzzing*.log - - ./gl-api_fuzzing*.zip - - $FUZZAPI_REPORT_ASSET_PATH - - $FUZZAPI_REPORT + - gl-assets + - gl-api-fuzzing-report.json + - gl-*.log reports: - api_fuzzing: $FUZZAPI_REPORT + api_fuzzing: gl-api-fuzzing-report.json # end diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 8fa33026011..0c4c39cbcd6 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: FUZZAPI_VERSION: "1" diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index c628e30b2c7..bd163f9db94 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -1,60 +1,44 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/container_scanning/ +# Use this template to enable container scanning in your project. +# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` +# keyword. +# The template should work without modifications but you can customize the template settings if +# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings +# +# Requirements: +# - A `test` stage to be present in the pipeline. +# - You must define the image to be scanned in the DOCKER_IMAGE variable. If DOCKER_IMAGE is the +# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. +# - Container registry credentials defined by `DOCKER_USER` and `DOCKER_PASSWORD` variables if the +# image to be scanned is in a private registry. +# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the +# DOCKERFILE_PATH variable. +# +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: - # Setting this variable will affect all Security templates - # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - CS_MAJOR_VERSION: 3 + CS_ANALYZER_IMAGE: registry.gitlab.com/security-products/container-scanning:4 -.cs_common: - stage: test +container_scanning: image: "$CS_ANALYZER_IMAGE" + stage: test variables: - # Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml` - # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template - # for details + # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your + # `.gitlab-ci.yml` file and set it to `fetch`. + # For details, see the following links: + # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template + # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting GIT_STRATEGY: none - # CS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/$CS_PROJECT:$CS_MAJOR_VERSION allow_failure: true artifacts: reports: container_scanning: gl-container-scanning-report.json + paths: [gl-container-scanning-report.json] dependencies: [] - -container_scanning: - extends: .cs_common - variables: - # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image - # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes - CLAIR_DB_IMAGE_TAG: "latest" - CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG" - CS_PROJECT: 'klar' - services: - - name: $CLAIR_DB_IMAGE - alias: clair-vulnerabilities-db - script: - - /analyzer run - rules: - - if: $CONTAINER_SCANNING_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && - $CS_MAJOR_VERSION =~ /^[0-3]$/ - -container_scanning_new: - extends: .cs_common - variables: - CS_PROJECT: 'container-scanning' script: - gtcs scan - artifacts: - paths: [gl-container-scanning-report.json] rules: - if: $CONTAINER_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && - $CS_MAJOR_VERSION !~ /^[0-3]$/ + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 9d47537c0f0..2dbfb80b419 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -1,5 +1,8 @@ # Read more about this feature https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing +# Configure coverage fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/#available-cicd-variables + variables: # Which branch we want to run full fledged long running fuzzing jobs. # All others will run fuzzing regression diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml index b40c4e982f7..9170e943e9d 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -13,9 +13,8 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html -# Configure the scanning tool with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html) -# List of variables available to configure the DAST API scanning tool: -# https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables +# Configure DAST API scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables variables: # Setting this variable affects all Security templates diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 7abecfb7e49..a2b112b8e9f 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables stages: - build @@ -11,7 +10,7 @@ stages: - dast variables: - DAST_VERSION: 1 + DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index b6282da18a4..6834766da3d 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -13,12 +13,11 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: - DAST_VERSION: 1 + DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" @@ -47,10 +46,13 @@ dast: $REVIEW_DISABLED && $DAST_WEBSITE == null && $DAST_API_SPECIFICATION == null when: never - - if: $CI_COMMIT_BRANCH && + - if: $CI_MERGE_REQUEST_IID && $CI_KUBERNETES_ACTIVE && $GITLAB_FEATURES =~ /\bdast\b/ + - if: $CI_MERGE_REQUEST_IID && ($DAST_WEBSITE || $DAST_API_SPECIFICATION) + - if: $CI_OPEN_MERGE_REQUESTS + when: never - if: $CI_COMMIT_BRANCH && - $DAST_WEBSITE - - if: $CI_COMMIT_BRANCH && - $DAST_API_SPECIFICATION + $CI_KUBERNETES_ACTIVE && + $GITLAB_FEATURES =~ /\bdast\b/ + - if: $CI_COMMIT_BRANCH && ($DAST_WEBSITE || $DAST_API_SPECIFICATION) diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 53d68c24d26..8df5ce79fe8 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/ # -# Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/dependency-scanning#settings -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure dependency scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/index.html#available-variables variables: # Setting this variable will affect all Security templates diff --git a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml deleted file mode 100644 index 87f78d0c887..00000000000 --- a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml +++ /dev/null @@ -1,13 +0,0 @@ -# Deprecated: https://gitlab.com/gitlab-org/gitlab/issues/14624 -# Please, use License-Scanning.gitlab-ci.yml template instead - -include: - - template: License-Scanning.gitlab-ci.yml - -license_scanning: - before_script: - - | - echo "As of GitLab 12.8, we deprecated the License-Management.gitlab.ci.yml template. - Please replace it with the License-Scanning.gitlab-ci.yml template instead. - For more details visit - https://docs.gitlab.com/ee/user/compliance/license_compliance/#migration-from-license_management-to-license_scanning" diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index 21e926ef275..870684c9f1d 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/compliance/license_compliance/index.html # -# Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/analyzers/license-finder#settings -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure license scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/compliance/license_compliance/#available-variables variables: # Setting this variable will affect all Security templates diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index a8d45e80356..77ce813dd4f 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -1,340 +1,5 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/ -# -# Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/sast#settings -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# This template moved to Jobs/SAST.gitlab-ci.yml in GitLab 14.0 +# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977 -variables: - # Setting this variable will affect all Security templates - # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - - SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf, semgrep" - SAST_EXCLUDED_ANALYZERS: "" - SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" - SAST_ANALYZER_IMAGE_TAG: 2 - SCAN_KUBERNETES_MANIFESTS: "false" - -sast: - stage: test - artifacts: - reports: - sast: gl-sast-report.json - rules: - - when: never - variables: - SEARCH_MAX_DEPTH: 4 - script: - - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" - - exit 1 - -.sast-analyzer: - extends: sast - allow_failure: true - # `rules` must be overridden explicitly by each child job - # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 - script: - - /analyzer run - -bandit-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /bandit/ - exists: - - '**/*.py' - -brakeman-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /brakeman/ - exists: - - '**/*.rb' - - '**/Gemfile' - -eslint-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /eslint/ - exists: - - '**/*.html' - - '**/*.js' - - '**/*.jsx' - - '**/*.ts' - - '**/*.tsx' - -flawfinder-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ - exists: - - '**/*.c' - - '**/*.cpp' - -kubesec-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && - $SCAN_KUBERNETES_MANIFESTS == 'true' - -gosec-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /gosec/ - exists: - - '**/*.go' - -.mobsf-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" - -mobsf-android-sast: - extends: .mobsf-sast - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && - $SAST_EXPERIMENTAL_FEATURES == 'true' - exists: - - '**/*.apk' - - '**/AndroidManifest.xml' - -mobsf-ios-sast: - extends: .mobsf-sast - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && - $SAST_EXPERIMENTAL_FEATURES == 'true' - exists: - - '**/*.ipa' - - '**/*.xcodeproj/*' - -nodejs-scan-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ - exists: - - '**/package.json' - -phpcs-security-audit-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ - exists: - - '**/*.php' - -pmd-apex-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ - exists: - - '**/*.cls' - -security-code-scan-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ - exists: - - '**/*.csproj' - - '**/*.vbproj' - -semgrep-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /semgrep/ - exists: - - '**/*.py' - - '**/*.js' - - '**/*.jsx' - - '**/*.ts' - - '**/*.tsx' - -sobelow-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /sobelow/ - exists: - - 'mix.exs' - -spotbugs-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ - when: never - - if: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && - $SAST_EXPERIMENTAL_FEATURES == 'true' - exists: - - '**/AndroidManifest.xml' - when: never - - if: $SAST_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ - exists: - - '**/*.groovy' - - '**/*.java' - - '**/*.scala' - - '**/*.kt' +include: + template: Jobs/SAST.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index c255fb4707a..d4ea7165d0a 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -1,45 +1,5 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection -# -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# This template moved to Jobs/Secret-Detection.gitlab-ci.yml in GitLab 14.0 +# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977 -variables: - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SECRETS_ANALYZER_VERSION: "3" - SECRET_DETECTION_EXCLUDED_PATHS: "" - - -.secret-analyzer: - stage: test - image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" - services: [] - allow_failure: true - # `rules` must be overridden explicitly by each child job - # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 - artifacts: - reports: - secret_detection: gl-secret-detection-report.json - -secret_detection_default_branch: - extends: .secret-analyzer - rules: - - if: $SECRET_DETECTION_DISABLED - when: never - - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH - script: - - /analyzer run - -secret_detection: - extends: .secret-analyzer - rules: - - if: $SECRET_DETECTION_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH - script: - - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME - - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt - - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt - - /analyzer run - - rm "$CI_COMMIT_SHA"_commit_list.txt +include: + template: Jobs/Secret-Detection.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index ac975fbbeab..d410c49b9a4 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -15,7 +15,6 @@ variables: SECURE_BINARIES_ANALYZERS: >- bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep, bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, - klar, clair-vulnerabilities-db, license-finder, dast, api-fuzzing @@ -78,6 +77,8 @@ brakeman: gosec: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -161,28 +162,6 @@ kubesec: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bkubesec\b/ -# -# Container Scanning jobs -# - -klar: - extends: .download_images - only: - variables: - - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && - $SECURE_BINARIES_ANALYZERS =~ /\bklar\b/ - variables: - SECURE_BINARIES_ANALYZER_VERSION: "3" - -clair-vulnerabilities-db: - extends: .download_images - only: - variables: - - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && - $SECURE_BINARIES_ANALYZERS =~ /\bclair-vulnerabilities-db\b/ - variables: - SECURE_BINARIES_IMAGE: arminc/clair-db - SECURE_BINARIES_ANALYZER_VERSION: latest # # Dependency Scanning jobs diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 6b9db1c2e0f..62b32d7c2db 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -1,61 +1,22 @@ -# Official image for Hashicorp's Terraform. It uses light image which is Alpine -# based as it is much lighter. -# -# Entrypoint is also needed as image by default set `terraform` binary as an -# entrypoint. -image: - name: registry.gitlab.com/gitlab-org/gitlab-build-images:terraform - entrypoint: - - '/usr/bin/env' - - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - -# Default output file for Terraform plan -variables: - PLAN: plan.tfplan - JSON_PLAN_FILE: tfplan.json - -cache: - paths: - - .terraform - - .terraform.lock.hcl - -before_script: - - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" - - terraform --version - - terraform init +include: + - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml stages: + - init - validate - build - - test - deploy +init: + extends: .init + validate: - stage: validate - script: - - terraform validate + extends: .validate -plan: - stage: build - script: - - terraform plan -out=$PLAN - - "terraform show --json $PLAN | convert_report > $JSON_PLAN_FILE" - artifacts: - paths: - - $PLAN - reports: - terraform: $JSON_PLAN_FILE +build: + extends: .build -# Separate apply job for manual launching Terraform as it can be destructive -# action. -apply: - stage: deploy - environment: - name: production - script: - - terraform apply -input=false $PLAN +deploy: + extends: .deploy dependencies: - - plan - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - when: manual + - build diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index 404d4a4c6db..f0621165f8a 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -6,7 +6,7 @@ stages: - deploy - performance -performance: +browser_performance: stage: performance image: docker:git variables: diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index 035ba52da84..536cf9bd8d8 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -1,22 +1,28 @@ -default: +publish: image: node:latest - - # Validate that the repository contains a package.json and extract a few values from it. - before_script: + stage: deploy + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_REF_NAME =~ /^v\d+\.\d+\.\d+.*$/ + changes: + - package.json + script: + # If no .npmrc if included in the repo, generate a temporary one that is configured to publish to GitLab's NPM registry - | - if [[ ! -f package.json ]]; then - echo "No package.json found! A package.json file is required to publish a package to GitLab's NPM registry." - echo 'For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#creating-a-project' - exit 1 + if [[ ! -f .npmrc ]]; then + echo 'No .npmrc found! Creating one now. Please review the following link for more information: https://docs.gitlab.com/ee/user/packages/npm_registry/index.html#project-level-npm-endpoint-1' + { + echo "@${CI_PROJECT_ROOT_NAMESPACE}:registry=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/npm/" + echo "${CI_API_V4_URL#http*:}/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=\${CI_JOB_TOKEN}" + } >> .npmrc fi + - echo "Created the following .npmrc:"; cat .npmrc + + # Extract a few values from package.json - NPM_PACKAGE_NAME=$(node -p "require('./package.json').name") - NPM_PACKAGE_VERSION=$(node -p "require('./package.json').version") -# Validate that the package name is properly scoped to the project's root namespace. -# For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention -validate_package_scope: - stage: build - script: + # Validate that the package name is properly scoped to the project's root namespace. + # For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention - | if [[ ! $NPM_PACKAGE_NAME =~ ^@$CI_PROJECT_ROOT_NAMESPACE/ ]]; then echo "Invalid package scope! Packages must be scoped in the root namespace of the project, e.g. \"@${CI_PROJECT_ROOT_NAMESPACE}/${CI_PROJECT_NAME}\"" @@ -24,36 +30,12 @@ validate_package_scope: exit 1 fi -# If no .npmrc if included in the repo, generate a temporary one to use during the publish step -# that is configured to publish to GitLab's NPM registry -create_npmrc: - stage: build - script: + # Compare the version in package.json to all published versions. + # If the package.json version has not yet been published, run `npm publish`. - | - if [[ ! -f .npmrc ]]; then - echo 'No .npmrc found! Creating one now. Please review the following link for more information: https://docs.gitlab.com/ee/user/packages/npm_registry/index.html#authenticating-with-a-ci-job-token' - - { - echo '@${CI_PROJECT_ROOT_NAMESPACE}:registry=${CI_SERVER_PROTOCOL}://${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/' - echo '//${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/packages/npm/:_authToken=${CI_JOB_TOKEN}' - echo '//${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN}' - } >> .npmrc - - fi - artifacts: - paths: - - .npmrc - -# Publish the package. If the version in package.json has not yet been published, it will be -# published to GitLab's NPM registry. If the version already exists, the publish command -# will fail and the existing package will not be updated. -publish_package: - stage: deploy - script: - - | - { - npm publish && + if [[ $(npm view "${NPM_PACKAGE_NAME}" versions) != *"'${NPM_PACKAGE_VERSION}'"* ]]; then + npm publish echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" - } || { - echo "No new version of ${NPM_PACKAGE_NAME} published. This is most likely because version ${NPM_PACKAGE_VERSION} already exists in GitLab's NPM registry." - } + else + echo "Version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} has already been published, so no new version has been published." + fi diff --git a/lib/gitlab/ci/templates/npm.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.latest.gitlab-ci.yml deleted file mode 100644 index 536cf9bd8d8..00000000000 --- a/lib/gitlab/ci/templates/npm.latest.gitlab-ci.yml +++ /dev/null @@ -1,41 +0,0 @@ -publish: - image: node:latest - stage: deploy - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_REF_NAME =~ /^v\d+\.\d+\.\d+.*$/ - changes: - - package.json - script: - # If no .npmrc if included in the repo, generate a temporary one that is configured to publish to GitLab's NPM registry - - | - if [[ ! -f .npmrc ]]; then - echo 'No .npmrc found! Creating one now. Please review the following link for more information: https://docs.gitlab.com/ee/user/packages/npm_registry/index.html#project-level-npm-endpoint-1' - { - echo "@${CI_PROJECT_ROOT_NAMESPACE}:registry=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/npm/" - echo "${CI_API_V4_URL#http*:}/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=\${CI_JOB_TOKEN}" - } >> .npmrc - fi - - echo "Created the following .npmrc:"; cat .npmrc - - # Extract a few values from package.json - - NPM_PACKAGE_NAME=$(node -p "require('./package.json').name") - - NPM_PACKAGE_VERSION=$(node -p "require('./package.json').version") - - # Validate that the package name is properly scoped to the project's root namespace. - # For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention - - | - if [[ ! $NPM_PACKAGE_NAME =~ ^@$CI_PROJECT_ROOT_NAMESPACE/ ]]; then - echo "Invalid package scope! Packages must be scoped in the root namespace of the project, e.g. \"@${CI_PROJECT_ROOT_NAMESPACE}/${CI_PROJECT_NAME}\"" - echo 'For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention' - exit 1 - fi - - # Compare the version in package.json to all published versions. - # If the package.json version has not yet been published, run `npm publish`. - - | - if [[ $(npm view "${NPM_PACKAGE_NAME}" versions) != *"'${NPM_PACKAGE_VERSION}'"* ]]; then - npm publish - echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" - else - echo "Version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} has already been published, so no new version has been published." - fi diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index c4757edf74e..84eb860a168 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -14,6 +14,8 @@ module Gitlab UPDATE_FREQUENCY_DEFAULT = 60.seconds UPDATE_FREQUENCY_WHEN_BEING_WATCHED = 3.seconds + LOAD_BALANCING_STICKING_NAMESPACE = 'ci/build/trace' + ArchiveError = Class.new(StandardError) AlreadyArchivedError = Class.new(StandardError) LockedError = Class.new(StandardError) @@ -296,25 +298,31 @@ module Gitlab read_trace_artifact(job) { job.job_artifacts_trace } end - ## - # Overridden in EE - # - def destroy_stream(job) + def destroy_stream(build) + if consistent_archived_trace?(build) + ::Gitlab::Database::LoadBalancing::Sticking + .stick(LOAD_BALANCING_STICKING_NAMESPACE, build.id) + end + yield end - ## - # Overriden in EE - # - def read_trace_artifact(job) + def read_trace_artifact(build) + if consistent_archived_trace?(build) + ::Gitlab::Database::LoadBalancing::Sticking + .unstick_or_continue_sticking(LOAD_BALANCING_STICKING_NAMESPACE, build.id) + end + yield end + def consistent_archived_trace?(build) + ::Feature.enabled?(:gitlab_ci_archived_trace_consistent_reads, build.project, default_enabled: false) + end + def being_watched_cache_key "gitlab:ci:trace:#{job.id}:watched" end end end end - -::Gitlab::Ci::Trace.prepend_mod_with('Gitlab::Ci::Trace') diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 7c2e39b1e53..9f24ba99201 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -229,13 +229,8 @@ module Gitlab def next_chunk @chunks_cache[chunk_index] = begin - if ::Ci::BuildTraceChunk.consistent_reads_enabled?(build) - ::Ci::BuildTraceChunk - .safe_find_or_create_by(build: build, chunk_index: chunk_index) - else - ::Ci::BuildTraceChunk - .new(build: build, chunk_index: chunk_index) - end + ::Ci::BuildTraceChunk + .safe_find_or_create_by(build: build, chunk_index: chunk_index) end end diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index ce9efbda7ea..fcd70634630 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -11,7 +11,6 @@ module Gitlab :streamed, # new trace data has been sent by a runner :chunked, # new trace chunk has been created :mutated, # trace has been mutated when removing secrets - :overwrite, # runner requested overwritting a build trace :accepted, # scheduled chunks for migration and responded with 202 :finalized, # all live build trace chunks have been persisted :discarded, # failed to persist live chunks before timeout diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index e2a8af9c26b..ef9ba1b73c7 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -24,6 +24,10 @@ module Gitlab self end + def compact + Collection.new(select { |variable| !variable.value.nil? }) + end + def concat(resources) return self if resources.nil? @@ -64,11 +68,19 @@ module Gitlab end def expand_value(value, keep_undefined: false) - value.gsub(ExpandVariables::VARIABLES_REGEXP) do + value.gsub(Item::VARIABLES_REGEXP) do match = Regexp.last_match - result = @variables_by_key[match[1] || match[2]]&.value - result ||= match[0] if keep_undefined - result + if match[:key] + # we matched variable + if variable = @variables_by_key[match[:key]] + variable.value + elsif keep_undefined + match[0] + end + else + # we escape sequence + match[0] + end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 77da2c4cb91..0217e6129ca 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -7,6 +7,9 @@ module Gitlab class Item include Gitlab::Utils::StrongMemoize + VARIABLES_REGEXP = /\$\$|%%|\$(?<key>[a-zA-Z_][a-zA-Z0-9_]*)|\${\g<key>?}|%\g<key>%/.freeze.freeze + VARIABLE_REF_CHARS = %w[$ %].freeze + def initialize(key:, value:, public: true, file: false, masked: false, raw: false) raise ArgumentError, "`#{key}` must be of type String or nil value, while it was: #{value.class}" unless value.is_a?(String) || value.nil? @@ -34,9 +37,9 @@ module Gitlab strong_memoize(:depends_on) do next if raw - next unless ExpandVariables.possible_var_reference?(value) + next unless self.class.possible_var_reference?(value) - value.scan(ExpandVariables::VARIABLES_REGEXP).map(&:first) + value.scan(VARIABLES_REGEXP).filter_map(&:last) end end @@ -64,6 +67,12 @@ module Gitlab end end + def self.possible_var_reference?(value) + return unless value + + VARIABLE_REF_CHARS.any? { |symbol| value.include?(symbol) } + end + def to_s return to_runner_variable.to_s unless depends_on diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index f96a6629849..15cc0c28296 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -111,6 +111,22 @@ module Gitlab @ci_config.variables_with_data end + def yaml_variables_for(job_name) + job = jobs[job_name] + + return [] unless job + + Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( + from: root_variables, + to: transform_to_yaml_variables(job[:job_variables]), + inheritance: job.fetch(:root_variables_inheritance, true) + ) + end + + def stage_for(job_name) + jobs.dig(job_name, :stage) + end + private def variables diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index b3dc59466ec..6159fb0a811 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -8,13 +8,13 @@ module Gitlab # LifecycleEvents lets Rails initializers register application startup hooks # that are sensitive to forking. For example, to defer the creation of # watchdog threads. This lets us abstract away the Unix process - # lifecycles of Unicorn, Sidekiq, Puma, Puma Cluster, etc. + # lifecycles of Sidekiq, Puma, Puma Cluster, etc. # # We have the following lifecycle events. # # - on_before_fork (on master process): # - # Unicorn/Puma Cluster: This will be called exactly once, + # Puma Cluster: This will be called exactly once, # on startup, before the workers are forked. This is # called in the PARENT/MASTER process. # @@ -22,7 +22,7 @@ module Gitlab # # - on_master_start (on master process): # - # Unicorn/Puma Cluster: This will be called exactly once, + # Puma Cluster: This will be called exactly once, # on startup, before the workers are forked. This is # called in the PARENT/MASTER process. # @@ -30,7 +30,7 @@ module Gitlab # # - on_before_blackout_period (on master process): # - # Unicorn/Puma Cluster: This will be called before a blackout + # Puma Cluster: This will be called before a blackout # period when performing graceful shutdown of master. # This is called on `master` process. # @@ -38,7 +38,7 @@ module Gitlab # # - on_before_graceful_shutdown (on master process): # - # Unicorn/Puma Cluster: This will be called before a graceful + # Puma Cluster: This will be called before a graceful # shutdown of workers starts happening, but after blackout period. # This is called on `master` process. # @@ -46,11 +46,6 @@ module Gitlab # # - on_before_master_restart (on master process): # - # Unicorn: This will be called before a new master is spun up. - # This is called on forked master before `execve` to become - # a new masterfor Unicorn. This means that this does not really - # affect old master process. - # # Puma Cluster: This will be called before a new master is spun up. # This is called on `master` process. # @@ -58,7 +53,7 @@ module Gitlab # # - on_worker_start (on worker process): # - # Unicorn/Puma Cluster: This is called in the worker process + # Puma Cluster: This is called in the worker process # exactly once before processing requests. # # Sidekiq/Puma Single: This is called immediately. @@ -114,7 +109,7 @@ module Gitlab end # - # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.) + # Lifecycle integration methods (called from puma.rb, etc.) # def do_worker_start call(:worker_start_hooks, @worker_start_hooks) @@ -167,9 +162,6 @@ module Gitlab # Sidekiq doesn't fork return false if Gitlab::Runtime.sidekiq? - # Unicorn always forks - return true if Gitlab::Runtime.unicorn? - # Puma sometimes forks return true if in_clustered_puma? diff --git a/lib/gitlab/cluster/mixins/unicorn_http_server.rb b/lib/gitlab/cluster/mixins/unicorn_http_server.rb deleted file mode 100644 index 440ed02a355..00000000000 --- a/lib/gitlab/cluster/mixins/unicorn_http_server.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Cluster - module Mixins - module UnicornHttpServer - def self.prepended(base) - unless base.method_defined?(:reexec) && base.method_defined?(:stop) - raise 'missing method Unicorn::HttpServer#reexec or Unicorn::HttpServer#stop' - end - end - - def reexec - Gitlab::Cluster::LifecycleEvents.do_before_graceful_shutdown - - super - end - - # The stop on non-graceful shutdown is executed twice: - # `#stop(false)` and `#stop`. - # - # The first stop will wipe-out all workers, so we need to check - # the flag and a list of workers - def stop(graceful = true) - if graceful && @workers.any? # rubocop:disable Gitlab/ModuleWithInstanceVariables - Gitlab::Cluster::LifecycleEvents.do_before_graceful_shutdown - end - - super - end - end - end - end -end diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index fd9f58a34f3..e634291f894 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -12,12 +12,9 @@ module Gitlab require 'puma_worker_killer' PumaWorkerKiller.config do |config| - # Note! ram is expressed in megabytes (whereas GITLAB_UNICORN_MEMORY_MAX is in bytes) - # Importantly RAM is for _all_workers (ie, the cluster), - # not each worker as is the case with GITLAB_UNICORN_MEMORY_MAX worker_count = puma_options[:workers] || 1 - # The Puma Worker Killer checks the total RAM used by both the master - # and worker processes. + # The Puma Worker Killer checks the total memory used by the cluster, + # i.e. both primary and worker processes. # https://github.com/schneems/puma_worker_killer/blob/v0.1.0/lib/puma_worker_killer/puma_memory.rb#L57 # # Additional memory is added when running in `development` diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index e42b174e085..d7b31946ab0 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -24,7 +24,7 @@ module Gitlab 'media_src' => "'self'", 'script_src' => "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com", 'style_src' => "'self' 'unsafe-inline'", - 'worker_src' => "'self'", + 'worker_src' => "'self' blob: data:", 'object_src' => "'none'", 'report_uri' => nil } @@ -79,6 +79,7 @@ module Gitlab append_to_directive(settings_hash, 'script_src', cdn_host) append_to_directive(settings_hash, 'style_src', cdn_host) + append_to_directive(settings_hash, 'font_src', cdn_host) end def self.append_to_directive(settings_hash, directive, text) diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb index 7559cd376bf..b309802f296 100644 --- a/lib/gitlab/cycle_analytics/stage_summary.rb +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -3,10 +3,9 @@ module Gitlab module CycleAnalytics class StageSummary - def initialize(project, from:, to: nil, current_user:) + def initialize(project, options:, current_user:) @project = project - @from = from - @to = to + @options = options @current_user = current_user end @@ -20,15 +19,15 @@ module Gitlab private def issue_stats - serialize(Summary::Issue.new(project: @project, from: @from, to: @to, current_user: @current_user)) + serialize(Summary::Issue.new(project: @project, options: @options, current_user: @current_user)) end def commit_stats - serialize(Summary::Commit.new(project: @project, from: @from, to: @to)) + serialize(Summary::Commit.new(project: @project, options: @options)) end def deployments_summary - @deployments_summary ||= Summary::Deploy.new(project: @project, from: @from, to: @to) + @deployments_summary ||= Summary::Deploy.new(project: @project, options: @options) end def deploy_stats @@ -39,8 +38,7 @@ module Gitlab serialize( Summary::DeploymentFrequency.new( deployments: deployments_summary.value.raw_value, - from: @from, - to: @to), + options: @options), with_unit: true ) end @@ -50,8 +48,7 @@ module Gitlab end def serialize(summary_object, with_unit: false) - AnalyticsSummarySerializer.new.represent( - summary_object, with_unit: with_unit) + AnalyticsSummarySerializer.new.represent(summary_object, with_unit: with_unit) end end end diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index 67ad75652b0..50a8f189df0 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -4,10 +4,9 @@ module Gitlab module CycleAnalytics module Summary class Base - def initialize(project:, from:, to: nil) + def initialize(project:, options:) @project = project - @from = from - @to = to + @options = options end def title diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb index 1dc9d5de966..fb55c3df869 100644 --- a/lib/gitlab/cycle_analytics/summary/commit.rb +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -21,7 +21,7 @@ module Gitlab def commits_count return unless ref - @commits_count ||= gitaly_commit_client.commit_count(ref, after: @from, before: @to) + @commits_count ||= gitaly_commit_client.commit_count(ref, after: @options[:from], before: @options[:to]) end def gitaly_commit_client diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index e5bf6ef616f..ea16226a865 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -16,7 +16,7 @@ module Gitlab def deployments_count DeploymentsFinder - .new(project: @project, finished_after: @from, finished_before: @to, status: :success, order_by: :finished_at) + .new(project: @project, finished_after: @options[:from], finished_before: @options[:to], status: :success, order_by: :finished_at) .execute .count end diff --git a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb index 00676a02a6f..1947866d772 100644 --- a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb +++ b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb @@ -6,10 +6,10 @@ module Gitlab class DeploymentFrequency < Base include SummaryHelper - def initialize(deployments:, from:, to: nil, project: nil) + def initialize(deployments:, options:, project: nil) @deployments = deployments - super(project: project, from: from, to: to) + super(project: project, options: options) end def title @@ -17,7 +17,7 @@ module Gitlab end def value - @value ||= frequency(@deployments, @from, @to || Time.now) + @value ||= frequency(@deployments, @options[:from], @options[:to] || Time.current) end def unit diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb index 462fd4c2d3d..34e0d34b960 100644 --- a/lib/gitlab/cycle_analytics/summary/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -4,10 +4,9 @@ module Gitlab module CycleAnalytics module Summary class Issue < Base - def initialize(project:, from:, to: nil, current_user:) + def initialize(project:, options:, current_user:) @project = project - @from = from - @to = to + @options = options @current_user = current_user end @@ -23,10 +22,18 @@ module Gitlab def issues_count IssuesFinder - .new(@current_user, project_id: @project.id, created_after: @from, created_before: @to) + .new(@current_user, finder_params) .execute .count end + + def finder_params + @options.dup.tap do |hash| + hash[:created_after] = hash.delete(:from) + hash[:created_before] = hash.delete(:to) + hash[:project_id] = @project.id + end + end end end end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 4c31f986be5..91e6fc11a53 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -83,7 +83,9 @@ module Gitlab { id: runner.id, description: runner.description, + runner_type: runner.runner_type, active: runner.active?, + is_shared: runner.instance_type?, tags: runner.tags&.map(&:name) } end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 766eaf54afe..4d70e3949dd 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -79,7 +79,9 @@ module Gitlab { id: runner.id, description: runner.description, + runner_type: runner.runner_type, active: runner.active?, + is_shared: runner.instance_type?, tags: runner.tags&.map(&:name) } end diff --git a/lib/gitlab/data_builder/wiki_page.rb b/lib/gitlab/data_builder/wiki_page.rb index 8aee25e9fe6..87679654a17 100644 --- a/lib/gitlab/data_builder/wiki_page.rb +++ b/lib/gitlab/data_builder/wiki_page.rb @@ -18,7 +18,8 @@ module Gitlab wiki: wiki.hook_attrs, object_attributes: wiki_page.hook_attrs.merge( url: Gitlab::UrlBuilder.build(wiki_page), - action: action + action: action, + diff_url: Gitlab::UrlBuilder.build(wiki_page, action: :diff, version_id: wiki_page.version.id) ) } end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 59249c8bc1f..aa419d75df2 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -9,12 +9,12 @@ module Gitlab # 'old_name' => 'new_name' # }.freeze TABLES_TO_BE_RENAMED = { - 'analytics_instance_statistics_measurements' => 'analytics_usage_trends_measurements' + 'services' => 'integrations' }.freeze # Minimum PostgreSQL version requirement per documentation: # https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements - MINIMUM_POSTGRES_VERSION = 11 + MINIMUM_POSTGRES_VERSION = 12 # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html MAX_INT_VALUE = 2147483647 @@ -60,7 +60,7 @@ module Gitlab end def self.config - default_config_hash = ActiveRecord::Base.configurations.find_db_config(Rails.env)&.config || {} + default_config_hash = ActiveRecord::Base.configurations.find_db_config(Rails.env)&.configuration_hash || {} default_config_hash.with_indifferent_access.tap do |hash| # Match config/initializers/database_config.rb @@ -88,6 +88,11 @@ module Gitlab end end + # Disables prepared statements for the current database connection. + def self.disable_prepared_statements + ActiveRecord::Base.establish_connection(config.merge(prepared_statements: false)) + end + # @deprecated def self.postgresql? adapter_name.casecmp('postgresql') == 0 @@ -142,7 +147,7 @@ module Gitlab is required for this version of GitLab. <% if Rails.env.development? || Rails.env.test? %> If using gitlab-development-kit, please find the relevant steps here: - https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/postgresql.md#upgrade-postgresql + https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#upgrade-postgresql <% end %> Please upgrade your environment to a supported PostgreSQL version, see https://docs.gitlab.com/ee/install/requirements.html#database for details. @@ -288,7 +293,7 @@ module Gitlab # @param [ActiveRecord::Connection] ar_connection # @return [String] def self.get_write_location(ar_connection) - use_new_load_balancer_query = Gitlab::Utils.to_boolean(ENV['USE_NEW_LOAD_BALANCER_QUERY'], default: false) + use_new_load_balancer_query = Gitlab::Utils.to_boolean(ENV['USE_NEW_LOAD_BALANCER_QUERY'], default: true) sql = if use_new_load_balancer_query <<~NEWSQL diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 869b97b8ac0..9a1dc4ee17d 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -30,7 +30,7 @@ module Gitlab scope :successful_in_execution_order, -> { where.not(finished_at: nil).succeeded.order(:finished_at) } - delegate :aborted?, :job_class, :table_name, :column_name, :job_arguments, + delegate :job_class, :table_name, :column_name, :job_arguments, to: :batched_migration, prefix: :migration attribute :pause_ms, :integer, default: 100 diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index e85162f355e..36e89023c86 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -14,12 +14,20 @@ module Gitlab class_name: 'Gitlab::Database::BackgroundMigration::BatchedJob', foreign_key: :batched_background_migration_id + validates :job_arguments, uniqueness: { + scope: [:job_class_name, :table_name, :column_name] + } + scope :queue_order, -> { order(id: :asc) } + scope :queued, -> { where(status: [:active, :paused]) } + scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do + where(job_class_name: job_class_name, table_name: table_name, column_name: column_name) + .where("job_arguments = ?", job_arguments.to_json) # rubocop:disable Rails/WhereEquals + end enum status: { paused: 0, active: 1, - aborted: 2, finished: 3, failed: 4 } @@ -30,6 +38,14 @@ module Gitlab active.queue_order.first end + def self.successful_rows_counts(migrations) + BatchedJob + .succeeded + .where(batched_background_migration_id: migrations) + .group(:batched_background_migration_id) + .sum(:batch_size) + end + def interval_elapsed?(variance: 0) return true unless last_job diff --git a/lib/gitlab/database/consistency.rb b/lib/gitlab/database/consistency.rb index e99ea7a3232..17c16640e4c 100644 --- a/lib/gitlab/database/consistency.rb +++ b/lib/gitlab/database/consistency.rb @@ -4,28 +4,18 @@ module Gitlab module Database ## # This class is used to make it possible to ensure read consistency in - # GitLab EE without the need of overriding a lot of methods / classes / + # GitLab without the need of overriding a lot of methods / classes / # classs. # - # This is a CE class that does nothing in CE, because database load - # balancing is EE-only feature, but you can still use it in CE. It will - # start ensuring read consistency once it is overridden in EE. - # - # Using this class in CE helps to avoid creeping discrepancy between CE / - # EE only to force usage of the primary database in EE. - # class Consistency ## - # In CE there is no database load balancing, so all reads are expected to - # be consistent by the ACID guarantees of a single PostgreSQL instance. - # - # This method is overridden in EE. + # Within the block, disable the database load balancing for calls that + # require read consistency after recent writes. # def self.with_read_consistency(&block) - yield + ::Gitlab::Database::LoadBalancing::Session + .current.use_primary(&block) end end end end - -::Gitlab::Database::Consistency.singleton_class.prepend_mod_with('Gitlab::Database::Consistency') diff --git a/lib/gitlab/database/dynamic_model_helpers.rb b/lib/gitlab/database/dynamic_model_helpers.rb index 892f8291780..7439591be99 100644 --- a/lib/gitlab/database/dynamic_model_helpers.rb +++ b/lib/gitlab/database/dynamic_model_helpers.rb @@ -11,6 +11,25 @@ module Gitlab self.inheritance_column = :_type_disabled end end + + def each_batch(table_name, scope: ->(table) { table.all }, of: 1000) + if transaction_open? + raise <<~MSG.squish + each_batch should not run inside a transaction, you can disable + transactions by calling disable_ddl_transaction! in the body of + your migration class + MSG + end + + scope.call(define_batchable_model(table_name)) + .each_batch(of: of) { |batch| yield batch } + end + + def each_batch_range(table_name, scope: ->(table) { table.all }, of: 1000) + each_batch(table_name, scope: scope, of: of) do |batch| + yield batch.pluck('MIN(id), MAX(id)').first + end + end end end end diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb new file mode 100644 index 00000000000..88743cd2e75 --- /dev/null +++ b/lib/gitlab/database/load_balancing.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # The exceptions raised for connection errors. + CONNECTION_ERRORS = if defined?(PG) + [ + PG::ConnectionBad, + PG::ConnectionDoesNotExist, + PG::ConnectionException, + PG::ConnectionFailure, + PG::UnableToSend, + # During a failover this error may be raised when + # writing to a primary. + PG::ReadOnlySqlTransaction + ].freeze + else + [].freeze + end + + ProxyNotConfiguredError = Class.new(StandardError) + + # The connection proxy to use for load balancing (if enabled). + def self.proxy + unless @proxy + Gitlab::ErrorTracking.track_exception( + ProxyNotConfiguredError.new( + "Attempting to access the database load balancing proxy, but it wasn't configured.\n" \ + "Did you forget to call '#{self.name}.configure_proxy'?" + )) + end + + @proxy + end + + # Returns a Hash containing the load balancing configuration. + def self.configuration + Gitlab::Database.config[:load_balancing] || {} + end + + # Returns the maximum replica lag size in bytes. + def self.max_replication_difference + (configuration['max_replication_difference'] || 8.megabytes).to_i + end + + # Returns the maximum lag time for a replica. + def self.max_replication_lag_time + (configuration['max_replication_lag_time'] || 60.0).to_f + end + + # Returns the interval (in seconds) to use for checking the status of a + # replica. + def self.replica_check_interval + (configuration['replica_check_interval'] || 60).to_f + end + + # Returns the additional hosts to use for load balancing. + def self.hosts + configuration['hosts'] || [] + end + + def self.service_discovery_enabled? + configuration.dig('discover', 'record').present? + end + + def self.service_discovery_configuration + conf = configuration['discover'] || {} + + { + nameserver: conf['nameserver'] || 'localhost', + port: conf['port'] || 8600, + record: conf['record'], + record_type: conf['record_type'] || 'A', + interval: conf['interval'] || 60, + disconnect_timeout: conf['disconnect_timeout'] || 120, + use_tcp: conf['use_tcp'] || false + } + end + + def self.pool_size + Gitlab::Database.config[:pool] + end + + # Returns true if load balancing is to be enabled. + def self.enable? + return false if Gitlab::Runtime.rake? + return false if Gitlab::Runtime.sidekiq? && !Gitlab::Utils.to_boolean(ENV['ENABLE_LOAD_BALANCING_FOR_SIDEKIQ'], default: false) + return false unless self.configured? + + true + end + + # Returns true if load balancing has been configured. Since + # Sidekiq does not currently use load balancing, we + # may want Web application servers to detect replication lag by + # posting the write location of the database if load balancing is + # configured. + def self.configured? + hosts.any? || service_discovery_enabled? + end + + def self.start_service_discovery + return unless service_discovery_enabled? + + ServiceDiscovery.new(service_discovery_configuration).start + end + + # Configures proxying of requests. + def self.configure_proxy(proxy = ConnectionProxy.new(hosts)) + @proxy = proxy + + # This hijacks the "connection" method to ensure both + # `ActiveRecord::Base.connection` and all models use the same load + # balancing proxy. + ActiveRecord::Base.singleton_class.prepend(ActiveRecordProxy) + end + + def self.active_record_models + ActiveRecord::Base.descendants + end + + DB_ROLES = [ + ROLE_PRIMARY = :primary, + ROLE_REPLICA = :replica, + ROLE_UNKNOWN = :unknown + ].freeze + + # Returns the role (primary/replica) of the database the connection is + # connecting to. At the moment, the connection can only be retrieved by + # Gitlab::Database::LoadBalancer#read or #read_write or from the + # ActiveRecord directly. Therefore, if the load balancer doesn't + # recognize the connection, this method returns the primary role + # directly. In future, we may need to check for other sources. + def self.db_role_for_connection(connection) + return ROLE_PRIMARY if !enable? || @proxy.blank? + + proxy.load_balancer.db_role_for_connection(connection) + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/active_record_proxy.rb b/lib/gitlab/database/load_balancing/active_record_proxy.rb new file mode 100644 index 00000000000..7763497e770 --- /dev/null +++ b/lib/gitlab/database/load_balancing/active_record_proxy.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Module injected into ActiveRecord::Base to allow hijacking of the + # "connection" method. + module ActiveRecordProxy + def connection + LoadBalancing.proxy + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb new file mode 100644 index 00000000000..3a09689a724 --- /dev/null +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +# rubocop:disable GitlabSecurity/PublicSend + +module Gitlab + module Database + module LoadBalancing + # Redirecting of ActiveRecord connections. + # + # The ConnectionProxy class redirects ActiveRecord connection requests to + # the right load balancer pool, depending on the type of query. + class ConnectionProxy + WriteInsideReadOnlyTransactionError = Class.new(StandardError) + READ_ONLY_TRANSACTION_KEY = :load_balacing_read_only_transaction + + attr_reader :load_balancer + + # These methods perform writes after which we need to stick to the + # primary. + STICKY_WRITES = %i( + delete + delete_all + insert + update + update_all + ).freeze + + NON_STICKY_READS = %i( + sanitize_limit + select + select_one + select_rows + quote_column_name + ).freeze + + # hosts - The hosts to use for load balancing. + def initialize(hosts = []) + @load_balancer = LoadBalancer.new(hosts) + end + + def select_all(arel, name = nil, binds = [], preparable: nil) + if arel.respond_to?(:locked) && arel.locked + # SELECT ... FOR UPDATE queries should be sent to the primary. + write_using_load_balancer(:select_all, [arel, name, binds], + sticky: true) + else + read_using_load_balancer(:select_all, [arel, name, binds]) + end + end + + NON_STICKY_READS.each do |name| + define_method(name) do |*args, &block| + read_using_load_balancer(name, args, &block) + end + end + + STICKY_WRITES.each do |name| + define_method(name) do |*args, &block| + write_using_load_balancer(name, args, sticky: true, &block) + end + end + + def transaction(*args, &block) + if current_session.fallback_to_replicas_for_ambiguous_queries? + track_read_only_transaction! + read_using_load_balancer(:transaction, args, &block) + else + write_using_load_balancer(:transaction, args, sticky: true, &block) + end + + ensure + untrack_read_only_transaction! + end + + # Delegates all unknown messages to a read-write connection. + def method_missing(name, *args, &block) + if current_session.fallback_to_replicas_for_ambiguous_queries? + read_using_load_balancer(name, args, &block) + else + write_using_load_balancer(name, args, &block) + end + end + + # Performs a read using the load balancer. + # + # name - The name of the method to call on a connection object. + def read_using_load_balancer(name, args, &block) + if current_session.use_primary? && + !current_session.use_replicas_for_read_queries? + @load_balancer.read_write do |connection| + connection.send(name, *args, &block) + end + else + @load_balancer.read do |connection| + connection.send(name, *args, &block) + end + end + end + + # Performs a write using the load balancer. + # + # name - The name of the method to call on a connection object. + # sticky - If set to true the session will stick to the master after + # the write. + def write_using_load_balancer(name, args, sticky: false, &block) + if read_only_transaction? + raise WriteInsideReadOnlyTransactionError, 'A write query is performed inside a read-only transaction' + end + + @load_balancer.read_write do |connection| + # Sticking has to be enabled before calling the method. Not doing so + # could lead to methods called in a block still being performed on a + # secondary instead of on a primary (when necessary). + current_session.write! if sticky + + connection.send(name, *args, &block) + end + end + + private + + def current_session + ::Gitlab::Database::LoadBalancing::Session.current + end + + def track_read_only_transaction! + Thread.current[READ_ONLY_TRANSACTION_KEY] = true + end + + def untrack_read_only_transaction! + Thread.current[READ_ONLY_TRANSACTION_KEY] = nil + end + + def read_only_transaction? + Thread.current[READ_ONLY_TRANSACTION_KEY] == true + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/host.rb b/lib/gitlab/database/load_balancing/host.rb new file mode 100644 index 00000000000..3e74b5ea727 --- /dev/null +++ b/lib/gitlab/database/load_balancing/host.rb @@ -0,0 +1,209 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # A single database host used for load balancing. + class Host + attr_reader :pool, :last_checked_at, :intervals, :load_balancer, :host, :port + + delegate :connection, :release_connection, :enable_query_cache!, :disable_query_cache!, :query_cache_enabled, to: :pool + + CONNECTION_ERRORS = + if defined?(PG) + [ + ActionView::Template::Error, + ActiveRecord::StatementInvalid, + PG::Error + ].freeze + else + [ + ActionView::Template::Error, + ActiveRecord::StatementInvalid + ].freeze + end + + # host - The address of the database. + # load_balancer - The LoadBalancer that manages this Host. + def initialize(host, load_balancer, port: nil) + @host = host + @port = port + @load_balancer = load_balancer + @pool = Database.create_connection_pool(LoadBalancing.pool_size, host, port) + @online = true + @last_checked_at = Time.zone.now + + interval = LoadBalancing.replica_check_interval + @intervals = (interval..(interval * 2)).step(0.5).to_a + end + + # Disconnects the pool, once all connections are no longer in use. + # + # timeout - The time after which the pool should be forcefully + # disconnected. + def disconnect!(timeout = 120) + start_time = Metrics::System.monotonic_time + + while (Metrics::System.monotonic_time - start_time) <= timeout + break if pool.connections.none?(&:in_use?) + + sleep(2) + end + + pool.disconnect! + end + + def offline! + LoadBalancing::Logger.warn( + event: :host_offline, + message: 'Marking host as offline', + db_host: @host, + db_port: @port + ) + + @online = false + @pool.disconnect! + end + + # Returns true if the host is online. + def online? + return @online unless check_replica_status? + + refresh_status + + if @online + LoadBalancing::Logger.info( + event: :host_online, + message: 'Host is online after replica status check', + db_host: @host, + db_port: @port + ) + else + LoadBalancing::Logger.warn( + event: :host_offline, + message: 'Host is offline after replica status check', + db_host: @host, + db_port: @port + ) + end + + @online + rescue *CONNECTION_ERRORS + offline! + false + end + + def refresh_status + @online = replica_is_up_to_date? + @last_checked_at = Time.zone.now + end + + def check_replica_status? + (Time.zone.now - last_checked_at) >= intervals.sample + end + + def replica_is_up_to_date? + replication_lag_below_threshold? || data_is_recent_enough? + end + + def replication_lag_below_threshold? + if (lag_time = replication_lag_time) + lag_time <= LoadBalancing.max_replication_lag_time + else + false + end + end + + # Returns true if the replica has replicated enough data to be useful. + def data_is_recent_enough? + # It's possible for a replica to not replay WAL data for a while, + # despite being up to date. This can happen when a primary does not + # receive any writes for a while. + # + # To prevent this from happening we check if the lag size (in bytes) + # of the replica is small enough for the replica to be useful. We + # only do this if we haven't replicated in a while so we only need + # to connect to the primary when truly necessary. + if (lag_size = replication_lag_size) + lag_size <= LoadBalancing.max_replication_difference + else + false + end + end + + # Returns the replication lag time of this secondary in seconds as a + # float. + # + # This method will return nil if no lag time could be calculated. + def replication_lag_time + row = query_and_release('SELECT EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))::float as lag') + + row['lag'].to_f if row.any? + end + + # Returns the number of bytes this secondary is lagging behind the + # primary. + # + # This method will return nil if no lag size could be calculated. + def replication_lag_size + location = connection.quote(primary_write_location) + row = query_and_release(<<-SQL.squish) + SELECT pg_wal_lsn_diff(#{location}, pg_last_wal_replay_lsn())::float + AS diff + SQL + + row['diff'].to_i if row.any? + rescue *CONNECTION_ERRORS + nil + end + + def primary_write_location + load_balancer.primary_write_location + ensure + load_balancer.release_primary_connection + end + + def database_replica_location + row = query_and_release(<<-SQL.squish) + SELECT pg_last_wal_replay_lsn()::text AS location + SQL + + row['location'] if row.any? + rescue *CONNECTION_ERRORS + nil + end + + # Returns true if this host has caught up to the given transaction + # write location. + # + # location - The transaction write location as reported by a primary. + def caught_up?(location) + string = connection.quote(location) + + # In case the host is a primary pg_last_wal_replay_lsn/pg_last_xlog_replay_location() returns + # NULL. The recovery check ensures we treat the host as up-to-date in + # such a case. + query = <<-SQL.squish + SELECT NOT pg_is_in_recovery() + OR pg_wal_lsn_diff(pg_last_wal_replay_lsn(), #{string}) >= 0 + AS result + SQL + + row = query_and_release(query) + + ::Gitlab::Utils.to_boolean(row['result']) + rescue *CONNECTION_ERRORS + false + end + + def query_and_release(sql) + connection.select_all(sql).first || {} + rescue StandardError + {} + ensure + release_connection + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/host_list.rb b/lib/gitlab/database/load_balancing/host_list.rb new file mode 100644 index 00000000000..24800012947 --- /dev/null +++ b/lib/gitlab/database/load_balancing/host_list.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # A list of database hosts to use for connections. + class HostList + # hosts - The list of secondary hosts to add. + def initialize(hosts = []) + @hosts = hosts.shuffle + @pools = Set.new + @index = 0 + @mutex = Mutex.new + @hosts_gauge = Gitlab::Metrics.gauge(:db_load_balancing_hosts, 'Current number of load balancing hosts') + + set_metrics! + update_pools + end + + def hosts + @mutex.synchronize { @hosts.dup } + end + + def shuffle + @mutex.synchronize do + unsafe_shuffle + end + end + + def length + @mutex.synchronize { @hosts.length } + end + + def host_names_and_ports + @mutex.synchronize { @hosts.map { |host| [host.host, host.port] } } + end + + def manage_pool?(pool) + @pools.include?(pool) + end + + def hosts=(hosts) + @mutex.synchronize do + @hosts = hosts + unsafe_shuffle + update_pools + end + + set_metrics! + end + + # Sets metrics before returning next host + def next + next_host.tap do |_| + set_metrics! + end + end + + private + + def unsafe_shuffle + @hosts = @hosts.shuffle + @index = 0 + end + + # Returns the next available host. + # + # Returns a Gitlab::Database::LoadBalancing::Host instance, or nil if no + # hosts were available. + def next_host + @mutex.synchronize do + break if @hosts.empty? + + started_at = @index + + loop do + host = @hosts[@index] + @index = (@index + 1) % @hosts.length + + break host if host.online? + + # Return nil once we have cycled through all hosts and none were + # available. + break if @index == started_at + end + end + end + + def set_metrics! + @hosts_gauge.set({}, @hosts.length) + end + + def update_pools + @pools = Set.new(@hosts.map(&:pool)) + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb new file mode 100644 index 00000000000..a833bb8491f --- /dev/null +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -0,0 +1,275 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Load balancing for ActiveRecord connections. + # + # Each host in the load balancer uses the same credentials as the primary + # database. + # + # This class *requires* that `ActiveRecord::Base.retrieve_connection` + # always returns a connection to the primary. + class LoadBalancer + CACHE_KEY = :gitlab_load_balancer_host + VALID_HOSTS_CACHE_KEY = :gitlab_load_balancer_valid_hosts + + attr_reader :host_list + + # hosts - The hostnames/addresses of the additional databases. + def initialize(hosts = []) + @host_list = HostList.new(hosts.map { |addr| Host.new(addr, self) }) + @connection_db_roles = {}.compare_by_identity + @connection_db_roles_count = {}.compare_by_identity + end + + # Yields a connection that can be used for reads. + # + # If no secondaries were available this method will use the primary + # instead. + def read(&block) + connection = nil + conflict_retried = 0 + + while host + ensure_caching! + + begin + connection = host.connection + track_connection_role(connection, ROLE_REPLICA) + + return yield connection + rescue StandardError => error + untrack_connection_role(connection) + + if serialization_failure?(error) + # This error can occur when a query conflicts. See + # https://www.postgresql.org/docs/current/static/hot-standby.html#HOT-STANDBY-CONFLICT + # for more information. + # + # In this event we'll cycle through the secondaries at most 3 + # times before using the primary instead. + will_retry = conflict_retried < @host_list.length * 3 + + LoadBalancing::Logger.warn( + event: :host_query_conflict, + message: 'Query conflict on host', + conflict_retried: conflict_retried, + will_retry: will_retry, + db_host: host.host, + db_port: host.port, + host_list_length: @host_list.length + ) + + if will_retry + conflict_retried += 1 + release_host + else + break + end + elsif connection_error?(error) + host.offline! + release_host + else + raise error + end + end + end + + LoadBalancing::Logger.warn( + event: :no_secondaries_available, + message: 'No secondaries were available, using primary instead', + conflict_retried: conflict_retried, + host_list_length: @host_list.length + ) + + read_write(&block) + ensure + untrack_connection_role(connection) + end + + # Yields a connection that can be used for both reads and writes. + def read_write + connection = nil + # In the event of a failover the primary may be briefly unavailable. + # Instead of immediately grinding to a halt we'll retry the operation + # a few times. + retry_with_backoff do + connection = ActiveRecord::Base.retrieve_connection + track_connection_role(connection, ROLE_PRIMARY) + + yield connection + end + ensure + untrack_connection_role(connection) + end + + # Recognize the role (primary/replica) of the database this connection + # is connecting to. If the connection is not issued by this load + # balancer, return nil + def db_role_for_connection(connection) + return @connection_db_roles[connection] if @connection_db_roles[connection] + return ROLE_REPLICA if @host_list.manage_pool?(connection.pool) + return ROLE_PRIMARY if connection.pool == ActiveRecord::Base.connection_pool + end + + # Returns a host to use for queries. + # + # Hosts are scoped per thread so that multiple threads don't + # accidentally re-use the same host + connection. + def host + RequestStore[CACHE_KEY] ||= current_host_list.next + end + + # Releases the host and connection for the current thread. + def release_host + if host = RequestStore[CACHE_KEY] + host.disable_query_cache! + host.release_connection + end + + RequestStore.delete(CACHE_KEY) + RequestStore.delete(VALID_HOSTS_CACHE_KEY) + end + + def release_primary_connection + ActiveRecord::Base.connection_pool.release_connection + end + + # Returns the transaction write location of the primary. + def primary_write_location + location = read_write do |connection| + ::Gitlab::Database.get_write_location(connection) + end + + return location if location + + raise 'Failed to determine the write location of the primary database' + end + + # Returns true if all hosts have caught up to the given transaction + # write location. + def all_caught_up?(location) + @host_list.hosts.all? { |host| host.caught_up?(location) } + end + + # Returns true if there was at least one host that has caught up with the given transaction. + # + # In case of a retry, this method also stores the set of hosts that have caught up. + def select_caught_up_hosts(location) + all_hosts = @host_list.hosts + valid_hosts = all_hosts.select { |host| host.caught_up?(location) } + + return false if valid_hosts.empty? + + # Hosts can come online after the time when this scan was done, + # so we need to remember the ones that can be used. If the host went + # offline, we'll just rely on the retry mechanism to use the primary. + set_consistent_hosts_for_request(HostList.new(valid_hosts)) + + # Since we will be using a subset from the original list, let's just + # pick a random host and mix up the original list to ensure we don't + # only end up using one replica. + RequestStore[CACHE_KEY] = valid_hosts.sample + @host_list.shuffle + + true + end + + # Returns true if there was at least one host that has caught up with the given transaction. + # Similar to `#select_caught_up_hosts`, picks a random host, to rotate replicas we use. + # Unlike `#select_caught_up_hosts`, does not iterate over all hosts if finds any. + def select_up_to_date_host(location) + all_hosts = @host_list.hosts.shuffle + host = all_hosts.find { |host| host.caught_up?(location) } + + return false unless host + + RequestStore[CACHE_KEY] = host + + true + end + + def set_consistent_hosts_for_request(hosts) + RequestStore[VALID_HOSTS_CACHE_KEY] = hosts + end + + # Yields a block, retrying it upon error using an exponential backoff. + def retry_with_backoff(retries = 3, time = 2) + retried = 0 + last_error = nil + + while retried < retries + begin + return yield + rescue StandardError => error + raise error unless connection_error?(error) + + # We need to release the primary connection as otherwise Rails + # will keep raising errors when using the connection. + release_primary_connection + + last_error = error + sleep(time) + retried += 1 + time **= 2 + end + end + + raise last_error + end + + def connection_error?(error) + case error + when ActiveRecord::StatementInvalid, ActionView::Template::Error + # After connecting to the DB Rails will wrap query errors using this + # class. + connection_error?(error.cause) + when *CONNECTION_ERRORS + true + else + # When PG tries to set the client encoding but fails due to a + # connection error it will raise a PG::Error instance. Catching that + # would catch all errors (even those we don't want), so instead we + # check for the message of the error. + error.message.start_with?('invalid encoding name:') + end + end + + def serialization_failure?(error) + if error.cause + serialization_failure?(error.cause) + else + error.is_a?(PG::TRSerializationFailure) + end + end + + private + + def ensure_caching! + host.enable_query_cache! unless host.query_cache_enabled + end + + def track_connection_role(connection, role) + @connection_db_roles[connection] = role + @connection_db_roles_count[connection] ||= 0 + @connection_db_roles_count[connection] += 1 + end + + def untrack_connection_role(connection) + return if connection.blank? || @connection_db_roles_count[connection].blank? + + @connection_db_roles_count[connection] -= 1 + if @connection_db_roles_count[connection] <= 0 + @connection_db_roles.delete(connection) + @connection_db_roles_count.delete(connection) + end + end + + def current_host_list + RequestStore[VALID_HOSTS_CACHE_KEY] || @host_list + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/logger.rb b/lib/gitlab/database/load_balancing/logger.rb new file mode 100644 index 00000000000..ee67ffcc99c --- /dev/null +++ b/lib/gitlab/database/load_balancing/logger.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'database_load_balancing' + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/rack_middleware.rb b/lib/gitlab/database/load_balancing/rack_middleware.rb new file mode 100644 index 00000000000..4734ff99bd3 --- /dev/null +++ b/lib/gitlab/database/load_balancing/rack_middleware.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Rack middleware to handle sticking when serving Rails requests. Grape + # API calls are handled separately as different API endpoints need to + # stick based on different objects. + class RackMiddleware + STICK_OBJECT = 'load_balancing.stick_object' + + # Unsticks or continues sticking the current request. + # + # This method also updates the Rack environment so #call can later + # determine if we still need to stick or not. + # + # env - The Rack environment. + # namespace - The namespace to use for sticking. + # id - The identifier to use for sticking. + def self.stick_or_unstick(env, namespace, id) + return unless LoadBalancing.enable? + + Sticking.unstick_or_continue_sticking(namespace, id) + + env[STICK_OBJECT] ||= Set.new + env[STICK_OBJECT] << [namespace, id] + end + + def initialize(app) + @app = app + end + + def call(env) + # Ensure that any state that may have run before the first request + # doesn't linger around. + clear + + unstick_or_continue_sticking(env) + + result = @app.call(env) + + stick_if_necessary(env) + + result + ensure + clear + end + + # Determine if we need to stick based on currently available user data. + # + # Typically this code will only be reachable for Rails requests as + # Grape data is not yet available at this point. + def unstick_or_continue_sticking(env) + namespaces_and_ids = sticking_namespaces_and_ids(env) + + namespaces_and_ids.each do |namespace, id| + Sticking.unstick_or_continue_sticking(namespace, id) + end + end + + # Determine if we need to stick after handling a request. + def stick_if_necessary(env) + namespaces_and_ids = sticking_namespaces_and_ids(env) + + namespaces_and_ids.each do |namespace, id| + Sticking.stick_if_necessary(namespace, id) + end + end + + def clear + load_balancer.release_host + Session.clear_session + end + + def load_balancer + LoadBalancing.proxy.load_balancer + end + + # Determines the sticking namespace and identifier based on the Rack + # environment. + # + # For Rails requests this uses warden, but Grape and others have to + # manually set the right environment variable. + def sticking_namespaces_and_ids(env) + warden = env['warden'] + + if warden && warden.user + [[:user, warden.user.id]] + elsif env[STICK_OBJECT].present? + env[STICK_OBJECT].to_a + else + [] + end + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/resolver.rb b/lib/gitlab/database/load_balancing/resolver.rb new file mode 100644 index 00000000000..a291080cc3d --- /dev/null +++ b/lib/gitlab/database/load_balancing/resolver.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require 'net/dns' +require 'resolv' + +module Gitlab + module Database + module LoadBalancing + class Resolver + UnresolvableNameserverError = Class.new(StandardError) + + def initialize(nameserver) + @nameserver = nameserver + end + + def resolve + address = ip_address || ip_address_from_hosts_file || + ip_address_from_dns + + unless address + raise UnresolvableNameserverError, + "could not resolve #{@nameserver}" + end + + address + end + + private + + def ip_address + IPAddr.new(@nameserver) + rescue IPAddr::InvalidAddressError + end + + def ip_address_from_hosts_file + ip = Resolv::Hosts.new.getaddress(@nameserver) + IPAddr.new(ip) + rescue Resolv::ResolvError + end + + def ip_address_from_dns + answer = Net::DNS::Resolver.start(@nameserver, Net::DNS::A).answer + return if answer.empty? + + answer.first.address + rescue Net::DNS::Resolver::NoResponseError + raise UnresolvableNameserverError, "no response from DNS server(s)" + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb new file mode 100644 index 00000000000..9b42b25be1c --- /dev/null +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -0,0 +1,187 @@ +# frozen_string_literal: true + +require 'net/dns' +require 'resolv' + +module Gitlab + module Database + module LoadBalancing + # Service discovery of secondary database hosts. + # + # Service discovery works by periodically looking up a DNS record. If the + # DNS record returns a new list of hosts, this class will update the load + # balancer with said hosts. Requests may continue to use the old hosts + # until they complete. + class ServiceDiscovery + attr_reader :interval, :record, :record_type, :disconnect_timeout + + MAX_SLEEP_ADJUSTMENT = 10 + + RECORD_TYPES = { + 'A' => Net::DNS::A, + 'SRV' => Net::DNS::SRV + }.freeze + + Address = Struct.new(:hostname, :port) do + def to_s + port ? "#{hostname}:#{port}" : hostname + end + + def <=>(other) + self.to_s <=> other.to_s + end + end + + # nameserver - The nameserver to use for DNS lookups. + # port - The port of the nameserver. + # record - The DNS record to look up for retrieving the secondaries. + # record_type - The type of DNS record to look up + # interval - The time to wait between lookups. + # disconnect_timeout - The time after which an old host should be + # forcefully disconnected. + # use_tcp - Use TCP instaed of UDP to look up resources + def initialize(nameserver:, port:, record:, record_type: 'A', interval: 60, disconnect_timeout: 120, use_tcp: false) + @nameserver = nameserver + @port = port + @record = record + @record_type = record_type_for(record_type) + @interval = interval + @disconnect_timeout = disconnect_timeout + @use_tcp = use_tcp + end + + def start + Thread.new do + loop do + interval = + begin + refresh_if_necessary + rescue StandardError => error + # Any exceptions that might occur should be reported to + # Sentry, instead of silently terminating this thread. + Gitlab::ErrorTracking.track_exception(error) + + Gitlab::AppLogger.error( + "Service discovery encountered an error: #{error.message}" + ) + + self.interval + end + + # We slightly randomize the sleep() interval. This should reduce + # the likelihood of _all_ processes refreshing at the same time, + # possibly putting unnecessary pressure on the DNS server. + sleep(interval + rand(MAX_SLEEP_ADJUSTMENT)) + end + end + end + + # Refreshes the hosts, but only if the DNS record returned a new list of + # addresses. + # + # The return value is the amount of time (in seconds) to wait before + # checking the DNS record for any changes. + def refresh_if_necessary + interval, from_dns = addresses_from_dns + + current = addresses_from_load_balancer + + replace_hosts(from_dns) if from_dns != current + + interval + end + + # Replaces all the hosts in the load balancer with the new ones, + # disconnecting the old connections. + # + # addresses - An Array of Address structs to use for the new hosts. + def replace_hosts(addresses) + old_hosts = load_balancer.host_list.hosts + + load_balancer.host_list.hosts = addresses.map do |addr| + Host.new(addr.hostname, load_balancer, port: addr.port) + end + + # We must explicitly disconnect the old connections, otherwise we may + # leak database connections over time. For example, if a request + # started just before we added the new hosts it will use an old + # host/connection. While this connection will be checked in and out, + # it won't be explicitly disconnected. + old_hosts.each do |host| + host.disconnect!(disconnect_timeout) + end + end + + # Returns an Array containing: + # + # 1. The time to wait for the next check. + # 2. An array containing the hostnames of the DNS record. + def addresses_from_dns + response = resolver.search(record, record_type) + resources = response.answer + + addresses = + case record_type + when Net::DNS::A + addresses_from_a_record(resources) + when Net::DNS::SRV + addresses_from_srv_record(response) + end + + # Addresses are sorted so we can directly compare the old and new + # addresses, without having to use any additional data structures. + [new_wait_time_for(resources), addresses.sort] + end + + def new_wait_time_for(resources) + wait = resources.first&.ttl || interval + + # The preconfigured interval acts as a minimum amount of time to + # wait. + wait < interval ? interval : wait + end + + def addresses_from_load_balancer + load_balancer.host_list.host_names_and_ports.map do |hostname, port| + Address.new(hostname, port) + end.sort + end + + def load_balancer + LoadBalancing.proxy.load_balancer + end + + def resolver + @resolver ||= Net::DNS::Resolver.new( + nameservers: Resolver.new(@nameserver).resolve, + port: @port, + use_tcp: @use_tcp + ) + end + + private + + def record_type_for(type) + RECORD_TYPES.fetch(type) do + raise(ArgumentError, "Unsupported record type: #{type}") + end + end + + def addresses_from_srv_record(response) + srv_resolver = SrvResolver.new(resolver, response.additional) + + response.answer.map do |r| + address = srv_resolver.address_for(r.host.to_s) + next unless address + + Address.new(address.to_s, r.port) + end.compact + end + + def addresses_from_a_record(resources) + resources.map { |r| Address.new(r.address.to_s) } + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/session.rb b/lib/gitlab/database/load_balancing/session.rb new file mode 100644 index 00000000000..3682c9265c2 --- /dev/null +++ b/lib/gitlab/database/load_balancing/session.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Tracking of load balancing state per user session. + # + # A session starts at the beginning of a request and ends once the request + # has been completed. Sessions can be used to keep track of what hosts + # should be used for queries. + class Session + CACHE_KEY = :gitlab_load_balancer_session + + def self.current + RequestStore[CACHE_KEY] ||= new + end + + def self.clear_session + RequestStore.delete(CACHE_KEY) + end + + def self.without_sticky_writes(&block) + current.ignore_writes(&block) + end + + def initialize + @use_primary = false + @performed_write = false + @ignore_writes = false + @fallback_to_replicas_for_ambiguous_queries = false + @use_replicas_for_read_queries = false + end + + def use_primary? + @use_primary + end + + alias_method :using_primary?, :use_primary? + + def use_primary! + @use_primary = true + end + + def use_primary(&blk) + used_primary = @use_primary + @use_primary = true + yield + ensure + @use_primary = used_primary || @performed_write + end + + def ignore_writes(&block) + @ignore_writes = true + + yield + ensure + @ignore_writes = false + end + + # Indicates that the read SQL statements from anywhere inside this + # blocks should use a replica, regardless of the current primary + # stickiness or whether a write query is already performed in the + # current session. This interface is reserved mostly for performance + # purpose. This is a good tool to push expensive queries, which can + # tolerate the replica lags, to the replicas. + # + # Write and ambiguous queries inside this block are still handled by + # the primary. + def use_replicas_for_read_queries(&blk) + previous_flag = @use_replicas_for_read_queries + @use_replicas_for_read_queries = true + yield + ensure + @use_replicas_for_read_queries = previous_flag + end + + def use_replicas_for_read_queries? + @use_replicas_for_read_queries == true + end + + # Indicate that the ambiguous SQL statements from anywhere inside this + # block should use a replica. The ambiguous statements include: + # - Transactions. + # - Custom queries (via exec_query, execute, etc.) + # - In-flight connection configuration change (SET LOCAL statement_timeout = 5000) + # + # This is a weak enforcement. This helper incorporates well with + # primary stickiness: + # - If the queries are about to write + # - The current session already performed writes + # - It prefers to use primary, aka, use_primary or use_primary! were called + def fallback_to_replicas_for_ambiguous_queries(&blk) + previous_flag = @fallback_to_replicas_for_ambiguous_queries + @fallback_to_replicas_for_ambiguous_queries = true + yield + ensure + @fallback_to_replicas_for_ambiguous_queries = previous_flag + end + + def fallback_to_replicas_for_ambiguous_queries? + @fallback_to_replicas_for_ambiguous_queries == true && !use_primary? && !performed_write? + end + + def write! + @performed_write = true + + return if @ignore_writes + + use_primary! + end + + def performed_write? + @performed_write + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb new file mode 100644 index 00000000000..524d69c00c0 --- /dev/null +++ b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + class SidekiqClientMiddleware + def call(worker_class, job, _queue, _redis_pool) + worker_class = worker_class.to_s.safe_constantize + + mark_data_consistency_location(worker_class, job) + + yield + end + + private + + def mark_data_consistency_location(worker_class, job) + # Mailers can't be constantized + return unless worker_class + return unless worker_class.include?(::ApplicationWorker) + return unless worker_class.get_data_consistency_feature_flag_enabled? + + return if location_already_provided?(job) + + job['worker_data_consistency'] = worker_class.get_data_consistency + + return unless worker_class.utilizes_load_balancing_capabilities? + + if Session.current.use_primary? + job['database_write_location'] = load_balancer.primary_write_location + else + job['database_replica_location'] = load_balancer.host.database_replica_location + end + end + + def location_already_provided?(job) + job['database_replica_location'] || job['database_write_location'] + end + + def load_balancer + LoadBalancing.proxy.load_balancer + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb new file mode 100644 index 00000000000..9bd0adf8dbd --- /dev/null +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + class SidekiqServerMiddleware + JobReplicaNotUpToDate = Class.new(StandardError) + + def call(worker, job, _queue) + if requires_primary?(worker.class, job) + Session.current.use_primary! + end + + yield + ensure + clear + end + + private + + def clear + load_balancer.release_host + Session.clear_session + end + + def requires_primary?(worker_class, job) + return true unless worker_class.include?(::ApplicationWorker) + return true unless worker_class.utilizes_load_balancing_capabilities? + return true unless worker_class.get_data_consistency_feature_flag_enabled? + + location = job['database_write_location'] || job['database_replica_location'] + + return true unless location + + job_data_consistency = worker_class.get_data_consistency + job[:data_consistency] = job_data_consistency.to_s + + if replica_caught_up?(location) + job[:database_chosen] = 'replica' + false + elsif job_data_consistency == :delayed && not_yet_retried?(job) + job[:database_chosen] = 'retry' + raise JobReplicaNotUpToDate, "Sidekiq job #{worker_class} JID-#{job['jid']} couldn't use the replica."\ + " Replica was not up to date." + else + job[:database_chosen] = 'primary' + true + end + end + + def not_yet_retried?(job) + # if `retry_count` is `nil` it indicates that this job was never retried + # the `0` indicates that this is a first retry + job['retry_count'].nil? + end + + def load_balancer + LoadBalancing.proxy.load_balancer + end + + def replica_caught_up?(location) + if Feature.enabled?(:sidekiq_load_balancing_rotate_up_to_date_replica) + load_balancer.select_up_to_date_host(location) + else + load_balancer.host.caught_up?(location) + end + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/srv_resolver.rb b/lib/gitlab/database/load_balancing/srv_resolver.rb new file mode 100644 index 00000000000..20da525f4d2 --- /dev/null +++ b/lib/gitlab/database/load_balancing/srv_resolver.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Hostnames returned in SRV records cannot sometimes be resolved by a local + # resolver, however, there's a possibility that their A/AAAA records are + # returned as part of the SRV query in the additional section, so we try + # to extract the IPs from there first, failing back to querying the + # hostnames A/AAAA records one by one, using the same resolver that + # queried the SRV record. + class SrvResolver + include Gitlab::Utils::StrongMemoize + + attr_reader :resolver, :additional + + def initialize(resolver, additional) + @resolver = resolver + @additional = additional + end + + def address_for(host) + addresses_from_additional[host] || resolve_host(host) + end + + private + + def addresses_from_additional + strong_memoize(:addresses_from_additional) do + additional.each_with_object({}) do |rr, h| + h[rr.name] = rr.address if rr.is_a?(Net::DNS::RR::A) || rr.is_a?(Net::DNS::RR::AAAA) + end + end + end + + def resolve_host(host) + record = resolver.search(host, Net::DNS::ANY).answer.find do |rr| + rr.is_a?(Net::DNS::RR::A) || rr.is_a?(Net::DNS::RR::AAAA) + end + + record&.address + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb new file mode 100644 index 00000000000..efbd7099300 --- /dev/null +++ b/lib/gitlab/database/load_balancing/sticking.rb @@ -0,0 +1,147 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Module used for handling sticking connections to a primary, if + # necessary. + # + # ## Examples + # + # Sticking a user to the primary: + # + # Sticking.stick_if_necessary(:user, current_user.id) + # + # To unstick if possible, or continue using the primary otherwise: + # + # Sticking.unstick_or_continue_sticking(:user, current_user.id) + module Sticking + # The number of seconds after which a session should stop reading from + # the primary. + EXPIRATION = 30 + + # Sticks to the primary if a write was performed. + def self.stick_if_necessary(namespace, id) + return unless LoadBalancing.enable? + + stick(namespace, id) if Session.current.performed_write? + end + + # Checks if we are caught-up with all the work + def self.all_caught_up?(namespace, id) + location = last_write_location_for(namespace, id) + + return true unless location + + load_balancer.all_caught_up?(location).tap do |caught_up| + unstick(namespace, id) if caught_up + end + end + + # Selects hosts that have caught up with the primary. This ensures + # atomic selection of the host to prevent the host list changing + # in another thread. + # + # Returns true if one host was selected. + def self.select_caught_up_replicas(namespace, id) + location = last_write_location_for(namespace, id) + + # Unlike all_caught_up?, we return false if no write location exists. + # We want to be sure we talk to a replica that has caught up for a specific + # write location. If no such location exists, err on the side of caution. + return false unless location + + load_balancer.select_caught_up_hosts(location).tap do |selected| + unstick(namespace, id) if selected + end + end + + # Sticks to the primary if necessary, otherwise unsticks an object (if + # it was previously stuck to the primary). + def self.unstick_or_continue_sticking(namespace, id) + Session.current.use_primary! unless all_caught_up?(namespace, id) + end + + # Select a replica that has caught up with the primary. If one has not been + # found, stick to the primary. + def self.select_valid_host(namespace, id) + replica_selected = select_caught_up_replicas(namespace, id) + + Session.current.use_primary! unless replica_selected + end + + # Starts sticking to the primary for the given namespace and id, using + # the latest WAL pointer from the primary. + def self.stick(namespace, id) + return unless LoadBalancing.enable? + + mark_primary_write_location(namespace, id) + Session.current.use_primary! + end + + def self.bulk_stick(namespace, ids) + return unless LoadBalancing.enable? + + with_primary_write_location do |location| + ids.each do |id| + set_write_location_for(namespace, id, location) + end + end + + Session.current.use_primary! + end + + def self.with_primary_write_location + return unless LoadBalancing.configured? + + # Load balancing could be enabled for the Web application server, + # but it's not activated for Sidekiq. We should update Redis with + # the write location just in case load balancing is being used. + location = + if LoadBalancing.enable? + load_balancer.primary_write_location + else + Gitlab::Database.get_write_location(ActiveRecord::Base.connection) + end + + return if location.blank? + + yield(location) + end + + def self.mark_primary_write_location(namespace, id) + with_primary_write_location do |location| + set_write_location_for(namespace, id, location) + end + end + + # Stops sticking to the primary. + def self.unstick(namespace, id) + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_key_for(namespace, id)) + end + end + + def self.set_write_location_for(namespace, id, location) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_key_for(namespace, id), location, ex: EXPIRATION) + end + end + + def self.last_write_location_for(namespace, id) + Gitlab::Redis::SharedState.with do |redis| + redis.get(redis_key_for(namespace, id)) + end + end + + def self.redis_key_for(namespace, id) + "database-load-balancing/write-location/#{namespace}/#{id}" + end + + def self.load_balancer + LoadBalancing.proxy.load_balancer + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 3a94e109d2a..d155abefdc8 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -5,7 +5,7 @@ module Gitlab module MigrationHelpers include Migrations::BackgroundMigrationHelpers include DynamicModelHelpers - include Migrations::RenameTableHelpers + include RenameTableHelpers # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS MAX_IDENTIFIER_NAME_LENGTH = 63 @@ -1091,6 +1091,25 @@ module Gitlab execute("DELETE FROM batched_background_migrations WHERE #{conditions}") end + def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:) + migration = Gitlab::Database::BackgroundMigration::BatchedMigration + .for_configuration(job_class_name, table_name, column_name, job_arguments).first + + configuration = { + job_class_name: job_class_name, + table_name: table_name, + column_name: column_name, + job_arguments: job_arguments + } + + if migration.nil? + Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" + elsif !migration.finished? + raise "Expected batched background migration for the given configuration to be marked as 'finished', " \ + "but it is '#{migration.status}': #{configuration}" + end + end + # Returns an Array containing the indexes for the given column def indexes_for(table, column) column = column.to_s diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 8d5ea652bfc..fa30ffb62f5 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -131,12 +131,51 @@ module Gitlab final_delay end + # Requeue pending jobs previously queued with #queue_background_migration_jobs_by_range_at_intervals + # + # This method is useful to schedule jobs that had previously failed. + # + # job_class_name - The background migration job class as a string + # delay_interval - The duration between each job's scheduled time + # batch_size - The maximum number of jobs to fetch to memory from the database. + def requeue_background_migration_jobs_by_range_at_intervals(job_class_name, delay_interval, batch_size: BATCH_SIZE, initial_delay: 0) + # To not overload the worker too much we enforce a minimum interval both + # when scheduling and performing jobs. + delay_interval = [delay_interval, BackgroundMigrationWorker.minimum_interval].max + + final_delay = 0 + job_counter = 0 + + jobs = Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: job_class_name) + jobs.each_batch(of: batch_size) do |job_batch| + job_batch.each do |job| + final_delay = initial_delay + delay_interval * job_counter + + migrate_in(final_delay, job_class_name, job.arguments) + + job_counter += 1 + end + end + + duration = initial_delay + delay_interval * job_counter + say <<~SAY + Scheduled #{job_counter} #{job_class_name} jobs with an interval of #{delay_interval} seconds. + + The migration is expected to take at least #{duration} seconds. Expect all jobs to have completed after #{Time.zone.now + duration}." + SAY + + duration + end + # Creates a batched background migration for the given table. A batched migration runs one job # at a time, computing the bounds of the next batch based on the current migration settings and the previous # batch bounds. Each job's execution status is tracked in the database as the migration runs. The given job # class must be present in the Gitlab::BackgroundMigration module, and the batch class (if specified) must be # present in the Gitlab::BackgroundMigration::BatchingStrategies module. # + # If migration with same job_class_name, table_name, column_name, and job_aruments already exists, this helper + # will log an warning and not create a new one. + # # job_class_name - The background migration job class as a string # batch_table_name - The name of the table the migration will batch over # batch_column_name - The name of the column the migration will batch over @@ -180,6 +219,13 @@ module Gitlab sub_batch_size: SUB_BATCH_SIZE ) + if Gitlab::Database::BackgroundMigration::BatchedMigration.for_configuration(job_class_name, batch_table_name, batch_column_name, job_arguments).exists? + Gitlab::AppLogger.warn "Batched background migration not enqueued because it already exists: " \ + "job_class_name: #{job_class_name}, table_name: #{batch_table_name}, column_name: #{batch_column_name}, " \ + "job_arguments: #{job_arguments.inspect}" + return + end + job_interval = BATCH_MIN_DELAY if job_interval < BATCH_MIN_DELAY batch_max_value ||= connection.select_value(<<~SQL) @@ -194,13 +240,13 @@ module Gitlab job_class_name: job_class_name, table_name: batch_table_name, column_name: batch_column_name, + job_arguments: job_arguments, interval: job_interval, min_value: batch_min_value, max_value: batch_max_value, batch_class_name: batch_class_name, batch_size: batch_size, sub_batch_size: sub_batch_size, - job_arguments: job_arguments, status: migration_status) # This guard is necessary since #total_tuple_count was only introduced schema-wise, diff --git a/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb b/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb index 906312478ac..88affaa9757 100644 --- a/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb +++ b/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# This patch will be included in the next Rails release: https://github.com/rails/rails/pull/42368 +raise 'This patch can be removed' if Rails::VERSION::MAJOR > 6 + # rubocop:disable Gitlab/ModuleWithInstanceVariables module Gitlab module Database diff --git a/lib/gitlab/database/postgresql_adapter/type_map_cache.rb b/lib/gitlab/database/postgresql_adapter/type_map_cache.rb new file mode 100644 index 00000000000..ff66d9115ab --- /dev/null +++ b/lib/gitlab/database/postgresql_adapter/type_map_cache.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# Caches loading of additional types from the DB +# https://github.com/rails/rails/blob/v6.0.3.2/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb#L521-L589 + +# rubocop:disable Gitlab/ModuleWithInstanceVariables + +module Gitlab + module Database + module PostgresqlAdapter + module TypeMapCache + extend ActiveSupport::Concern + + TYPE_MAP_CACHE_MONITOR = ::Monitor.new + + class_methods do + def type_map_cache + TYPE_MAP_CACHE_MONITOR.synchronize do + @type_map_cache ||= {} + end + end + end + + def initialize_type_map(map = type_map) + TYPE_MAP_CACHE_MONITOR.synchronize do + cached_type_map = self.class.type_map_cache[@connection_parameters.hash] + break @type_map = cached_type_map if cached_type_map + + super + self.class.type_map_cache[@connection_parameters.hash] = map + end + end + + def reload_type_map + TYPE_MAP_CACHE_MONITOR.synchronize do + self.class.type_map_cache[@connection_parameters.hash] = nil + end + + super + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 9ed03c05f0b..f3f0f227a8c 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -19,6 +19,7 @@ module Gitlab @diffable = diffable @include_stats = diff_options.delete(:include_stats) + @pagination_data = diff_options.delete(:pagination_data) @project = project @diff_options = diff_options @diff_refs = diff_refs @@ -47,11 +48,7 @@ module Gitlab end def pagination_data - { - current_page: nil, - next_page: nil, - total_pages: nil - } + @pagination_data || empty_pagination_data end # This mutates `diff_files` lines. @@ -90,6 +87,14 @@ module Gitlab private + def empty_pagination_data + { + current_page: nil, + next_page: nil, + total_pages: nil + } + end + def diff_stats_collection strong_memoize(:diff_stats) do next unless fetch_diff_stats? diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb index 64523f3b730..5ff7c88970c 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb @@ -21,9 +21,9 @@ module Gitlab @paginated_collection = load_paginated_collection(batch_page, batch_size, diff_options) @pagination_data = { - current_page: batch_gradual_load? ? nil : @paginated_collection.current_page, - next_page: batch_gradual_load? ? nil : @paginated_collection.next_page, - total_pages: batch_gradual_load? ? relation.size : @paginated_collection.total_pages + current_page: current_page, + next_page: next_page, + total_pages: total_pages } end @@ -62,6 +62,24 @@ module Gitlab @merge_request_diff.merge_request_diff_files end + def current_page + return if @paginated_collection.blank? + + batch_gradual_load? ? nil : @paginated_collection.current_page + end + + def next_page + return if @paginated_collection.blank? + + batch_gradual_load? ? nil : @paginated_collection.next_page + end + + def total_pages + return if @paginated_collection.blank? + + batch_gradual_load? ? relation.size : @paginated_collection.total_pages + end + # rubocop: disable CodeReuse/ActiveRecord def load_paginated_collection(batch_page, batch_size, diff_options) batch_page ||= DEFAULT_BATCH_PAGE diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 6a41ed0f29e..32ce35110f8 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -70,12 +70,6 @@ module Gitlab return rich_line if marker_ranges.blank? begin - # MarkerRange objects are converted to Ranges to keep the previous behavior - # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/324068 - if Feature.disabled?(:introduce_marker_ranges, project, default_enabled: :yaml) - marker_ranges = marker_ranges.map { |marker_range| marker_range.to_range } - end - InlineDiffMarker.new(diff_line.text, rich_line).mark(marker_ranges) # This should only happen when the encoding of the diff doesn't # match the blob, which is a bug. But we shouldn't fail to render diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 209462fd6e9..a792eafde79 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -74,7 +74,6 @@ module Gitlab diffable.cache_key, VERSION, diff_options, - Feature.enabled?(:introduce_marker_ranges, diffable.project, default_enabled: :yaml), Feature.enabled?(:use_marker_ranges, diffable.project, default_enabled: :yaml), Feature.enabled?(:diff_line_syntax_highlighting, diffable.project, default_enabled: :yaml) ].join(":") diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index 63334169c8e..fd3143488b1 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -84,6 +84,8 @@ module Gitlab end def valid_project_slug?(found_project) + return false unless found_project + project_slug == found_project.full_path_slug end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index cab3538a447..05daa08530e 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -65,10 +65,9 @@ module Gitlab def project_from_key return unless match = service_desk_key.match(PROJECT_KEY_PATTERN) - project = Project.find_by_service_desk_project_key(match[:key]) - return unless valid_project_key?(project, match[:slug]) - - project + Project.with_service_desk_key(match[:key]).find do |project| + valid_project_key?(project, match[:slug]) + end end def valid_project_key?(project, slug) diff --git a/lib/gitlab/email/message/in_product_marketing.rb b/lib/gitlab/email/message/in_product_marketing.rb index d538238f26f..fb4315e74b2 100644 --- a/lib/gitlab/email/message/in_product_marketing.rb +++ b/lib/gitlab/email/message/in_product_marketing.rb @@ -6,10 +6,8 @@ module Gitlab module InProductMarketing UnknownTrackError = Class.new(StandardError) - TRACKS = [:create, :verify, :team, :trial].freeze - def self.for(track) - raise UnknownTrackError unless TRACKS.include?(track) + raise UnknownTrackError unless Namespaces::InProductMarketingEmailsService::TRACKS.key?(track) "Gitlab::Email::Message::InProductMarketing::#{track.to_s.classify}".constantize end diff --git a/lib/gitlab/email/message/in_product_marketing/base.rb b/lib/gitlab/email/message/in_product_marketing/base.rb index 6341a7c7596..89acc058a46 100644 --- a/lib/gitlab/email/message/in_product_marketing/base.rb +++ b/lib/gitlab/email/message/in_product_marketing/base.rb @@ -10,10 +10,11 @@ module Gitlab attr_accessor :format - def initialize(group:, series:, format: :html) + def initialize(group:, user:, series:, format: :html) raise ArgumentError, "Only #{total_series} series available for this track." unless series.between?(0, total_series - 1) @group = group + @user = user @series = series @format = format end @@ -103,11 +104,7 @@ module Gitlab protected - attr_reader :group, :series - - def total_series - 3 - end + attr_reader :group, :user, :series private @@ -115,6 +112,10 @@ module Gitlab self.class.name.demodulize.downcase.to_sym end + def total_series + Namespaces::InProductMarketingEmailsService::TRACKS[track][:interval_days].size + end + def unsubscribe_com [ s_('InProductMarketing|If you no longer wish to receive marketing emails from us,'), diff --git a/lib/gitlab/email/message/in_product_marketing/experience.rb b/lib/gitlab/email/message/in_product_marketing/experience.rb new file mode 100644 index 00000000000..4156a737517 --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/experience.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class Experience < Base + include Gitlab::Utils::StrongMemoize + + EASE_SCORE_SURVEY_ID = 1 + + def subject_line + s_('InProductMarketing|Do you have a minute?') + end + + def tagline + end + + def title + s_('InProductMarketing|We want your GitLab experience to be great') + end + + def subtitle + s_('InProductMarketing|Take this 1-question survey!') + end + + def body_line1 + s_('InProductMarketing|%{strong_start}Overall, how difficult or easy was it to get started with GitLab?%{strong_end}').html_safe % strong_options + end + + def body_line2 + s_('InProductMarketing|Click on the number below that corresponds with your answer — 1 being very difficult, 5 being very easy.') + end + + def cta_text + end + + def feedback_link(rating) + params = { + onboarding_progress: onboarding_progress, + response: rating, + show_invite_link: show_invite_link, + survey_id: EASE_SCORE_SURVEY_ID + } + + "#{Gitlab::Saas.com_url}/-/survey_responses?#{params.to_query}" + end + + def feedback_ratings(rating) + [ + s_('InProductMarketing|Very difficult'), + s_('InProductMarketing|Difficult'), + s_('InProductMarketing|Neutral'), + s_('InProductMarketing|Easy'), + s_('InProductMarketing|Very easy') + ][rating - 1] + end + + def feedback_thanks + s_('InProductMarketing|Feedback from users like you really improves our product. Thanks for your help!') + end + + private + + def onboarding_progress + strong_memoize(:onboarding_progress) do + group.onboarding_progress.number_of_completed_actions + end + end + + def show_invite_link + strong_memoize(:show_invite_link) do + group.member_count > 1 && group.max_member_access_for_user(user) >= GroupMember::DEVELOPER && user.preferred_language == 'en' + end + end + end + end + end + end +end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 71db8ab6067..8139a294269 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -20,7 +20,7 @@ module Gitlab raise UnknownIncomingEmail unless handler handler.execute.tap do - Gitlab::Metrics.add_event(handler.metrics_event, handler.metrics_params) + Gitlab::Metrics::BackgroundTransaction.current&.add_event(handler.metrics_event, handler.metrics_params) end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index e6f71e3ad3c..2b5f465d3c5 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -41,7 +41,17 @@ module Gitlab end def emoji_image_tag(name, src) - "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{src}' height='20' width='20' align='absmiddle' />" + image_options = { + class: 'emoji', + src: src, + title: ":#{name}:", + alt: ":#{name}:", + height: 20, + width: 20, + align: 'absmiddle' + } + + ActionController::Base.helpers.tag(:img, image_options) end def emoji_exists?(name) diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index e91488c7c27..38ac5d9af74 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -146,9 +146,6 @@ module Gitlab else inject_context_for_exception(event, ex.cause) if ex.cause.present? end - # This should only happen on PostgreSQL v12 queries - rescue PgQuery::ParseError - event.extra[:sql] = ex.sql.to_s end end end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 8c916375a98..d5bf0cffb1e 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -67,10 +67,11 @@ module Gitlab add_instrument_for_cache_hit(status_code, route, request) + Gitlab::ApplicationContext.push(feature_category: route.feature_category) + new_headers = { 'ETag' => etag, - 'X-Gitlab-From-Cache' => 'true', - ::Gitlab::Metrics::RequestsRackMiddleware::FEATURE_CATEGORY_HEADER => route.feature_category + 'X-Gitlab-From-Cache' => 'true' } [status_code, new_headers, []] diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb index da5b0afad38..7cf0232fbf2 100644 --- a/lib/gitlab/exclusive_lease_helpers.rb +++ b/lib/gitlab/exclusive_lease_helpers.rb @@ -25,7 +25,7 @@ module Gitlab # a proc that computes the sleep time given the number of preceding attempts # (from 1 to retries - 1) # - # Note: It's basically discouraged to use this method in a unicorn thread, + # Note: It's basically discouraged to use this method in a webserver thread, # because this ties up all thread related resources until all `retries` are consumed. # This could potentially eat up all connection pools. def in_lock(key, ttl: 1.minute, retries: 10, sleep_sec: 0.01.seconds) diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index e4233b8a935..fe3dd4759d6 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -45,12 +45,6 @@ module Gitlab remove_known_trial_form_fields: { tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFields' }, - invite_members_empty_project_version_a: { - tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyProjectVersionA' - }, - trial_during_signup: { - tracking_category: 'Growth::Conversion::Experiment::TrialDuringSignup' - }, invite_members_new_dropdown: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown' }, @@ -62,10 +56,12 @@ module Gitlab tracking_category: 'Growth::Conversion::Experiment::TrialOnboardingIssues' }, learn_gitlab_a: { - tracking_category: 'Growth::Conversion::Experiment::LearnGitLabA' + tracking_category: 'Growth::Conversion::Experiment::LearnGitLabA', + rollout_strategy: :user }, learn_gitlab_b: { - tracking_category: 'Growth::Activation::Experiment::LearnGitLabB' + tracking_category: 'Growth::Activation::Experiment::LearnGitLabB', + rollout_strategy: :user }, in_product_marketing_emails: { tracking_category: 'Growth::Activation::Experiment::InProductMarketingEmails' diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index e53689eb89b..ca9205a8f8c 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -56,7 +56,7 @@ module Gitlab return if dnt_enabled? track_experiment_event_for(experiment_key, action, value, subject: subject) do |tracking_data| - ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data) + ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data.merge!(user: current_user)) end end diff --git a/lib/gitlab/file_hook.rb b/lib/gitlab/file_hook.rb index e398a3f9585..a8719761278 100644 --- a/lib/gitlab/file_hook.rb +++ b/lib/gitlab/file_hook.rb @@ -11,7 +11,7 @@ module Gitlab end def self.dir_glob - Dir.glob([Rails.root.join('file_hooks/*'), Rails.root.join('plugins/*')]) + Dir.glob(Rails.root.join('file_hooks/*')) end private_class_method :dir_glob diff --git a/lib/gitlab/file_hook_logger.rb b/lib/gitlab/file_hook_logger.rb index c5e69172016..4d6a650161f 100644 --- a/lib/gitlab/file_hook_logger.rb +++ b/lib/gitlab/file_hook_logger.rb @@ -3,7 +3,7 @@ module Gitlab class FileHookLogger < Gitlab::Logger def self.file_name_noext - 'plugin' + 'file_hook' end end end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 751184b23df..aa5d50d1fb1 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -18,9 +18,9 @@ module Gitlab def conflicts @conflicts ||= wrapped_gitaly_errors do gitaly_conflicts_client(@target_repository).list_conflict_files.to_a + rescue GRPC::FailedPrecondition => e + raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing, e.message end - rescue GRPC::FailedPrecondition => e - raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing, e.message rescue GRPC::BadStatus => e raise Gitlab::Git::CommandError, e end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index fb947c80b7e..631624c068c 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -12,11 +12,7 @@ module Gitlab delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits def self.default_limits(project: nil) - if Feature.enabled?(:increased_diff_limits, project) - { max_files: 300, max_lines: 10000 } - else - { max_files: 100, max_lines: 5000 } - end + { max_files: ::Commit.diff_safe_max_files(project: project), max_lines: ::Commit.diff_safe_max_lines(project: project) } end def self.limits(options = {}) diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index a8d1ea08275..6c4191ce25b 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -3,13 +3,13 @@ module Gitlab module Git class LfsChanges - def initialize(repository, newrev = nil) + def initialize(repository, newrevs = nil) @repository = repository - @newrev = newrev + @newrevs = newrevs end def new_pointers(object_limit: nil, not_in: nil, dynamic_timeout: nil) - @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in, dynamic_timeout) + @repository.gitaly_blob_client.get_new_lfs_pointers(@newrevs, object_limit, not_in, dynamic_timeout) end def all_pointers diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb index 234541d8145..0ea009930b0 100644 --- a/lib/gitlab/git/remote_repository.rb +++ b/lib/gitlab/git/remote_repository.rb @@ -53,23 +53,6 @@ module Gitlab gitaly_repository.relative_path == other_repository.relative_path end - def fetch_env - gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh')) - gitaly_address = gitaly_client.address(storage) - gitaly_token = gitaly_client.token(storage) - - request = Gitaly::SSHUploadPackRequest.new(repository: gitaly_repository) - env = { - 'GITALY_ADDRESS' => gitaly_address, - 'GITALY_PAYLOAD' => request.to_json, - 'GITALY_WD' => Dir.pwd, - 'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack" - } - env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present? - - env - end - def path @repository.path end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 102fe60f2cb..e38c7b516ee 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -302,8 +302,6 @@ module Gitlab private :archive_file_path def archive_version_path - return '' unless Feature.enabled?(:include_lfs_blobs_in_archive, default_enabled: true) - '@v2' end private :archive_version_path @@ -797,15 +795,19 @@ module Gitlab # Fetch remote for repository # # remote - remote name + # url - URL of the remote to fetch. `remote` is not used in this case. + # refmap - if url is given, determines which references should get fetched where # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication # forced - should we use --force flag? # no_tags - should we use --no-tags flag? # prune - should we use --prune flag? # check_tags_changed - should we ask gitaly to calculate whether any tags changed? - def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false, prune: true, check_tags_changed: false) + def fetch_remote(remote, url: nil, refmap: nil, ssh_auth: nil, forced: false, no_tags: false, prune: true, check_tags_changed: false) wrapped_gitaly_errors do gitaly_repository_client.fetch_remote( remote, + url: url, + refmap: refmap, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index b5e7220889e..b2a65d9f2d8 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -334,23 +334,15 @@ module Gitlab # clear stale lock files. project.repository.clean_stale_repository_files if project.present? - # Iterate over all changes to find if user allowed all of them to be applied - changes_list.each.with_index do |change, index| - first_change = index == 0 - - # If user does not have access to make at least one change, cancel all - # push by allowing the exception to bubble up - check_single_change_access(change, skip_lfs_integrity_check: !first_change) - end + check_access! end end - def check_single_change_access(change, skip_lfs_integrity_check: false) - Checks::ChangeAccess.new( - change, + def check_access! + Checks::ChangesAccess.new( + changes_list.changes, user_access: user_access, project: project, - skip_lfs_integrity_check: skip_lfs_integrity_check, protocol: protocol, logger: logger ).validate! diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index 9a431dc7088..4d87b91764a 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -109,20 +109,18 @@ module Gitlab end check_size_before_push! + check_access! + check_push_size! + end + override :check_access! + def check_access! changes_list.each do |change| # If user does not have access to make at least one change, cancel all # push by allowing the exception to bubble up - check_single_change_access(change) + Checks::SnippetCheck.new(change, default_branch: snippet.default_branch, root_ref: snippet.repository.root_ref, logger: logger).validate! + Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet.max_file_limit, logger: logger).validate! end - - check_push_size! - end - - override :check_single_change_access - def check_single_change_access(change, _skip_lfs_integrity_check: false) - Checks::SnippetCheck.new(change, default_branch: snippet.default_branch, root_ref: snippet.repository.root_ref, logger: logger).validate! - Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet.max_file_limit, logger: logger).validate! rescue Checks::TimedLogger::TimeoutError raise TimeoutError, logger.full_message end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index affd3986381..e4c8dc150a5 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -77,8 +77,8 @@ module Gitlab map_blob_types(response) end - def get_new_lfs_pointers(revision, limit, not_in, dynamic_timeout = nil) - request, rpc = create_new_lfs_pointers_request(revision, limit, not_in) + def get_new_lfs_pointers(revisions, limit, not_in, dynamic_timeout = nil) + request, rpc = create_new_lfs_pointers_request(revisions, limit, not_in) timeout = if dynamic_timeout @@ -109,7 +109,7 @@ module Gitlab private - def create_new_lfs_pointers_request(revision, limit, not_in) + def create_new_lfs_pointers_request(revisions, limit, not_in) # If the check happens for a change which is using a quarantine # environment for incoming objects, then we can avoid doing the # necessary graph walk to detect only new LFS pointers and instead scan @@ -126,7 +126,7 @@ module Gitlab [request, :list_all_lfs_pointers] else - revisions = [revision] + revisions = Array.wrap(revisions) revisions += if not_in.nil? || not_in == :all ["--not", "--all"] else diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 04dd394a2bd..1f360385111 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -45,18 +45,9 @@ module Gitlab # The remote_name parameter is deprecated and will be removed soon. def find_remote_root_ref(remote_name, remote_url, authorization) - request = if Feature.enabled?(:find_remote_root_refs_inmemory, default_enabled: :yaml) - Gitaly::FindRemoteRootRefRequest.new( - repository: @gitaly_repo, - remote_url: remote_url, - http_authorization_header: authorization - ) - else - Gitaly::FindRemoteRootRefRequest.new( - repository: @gitaly_repo, - remote: remote_name - ) - end + request = Gitaly::FindRemoteRootRefRequest.new(repository: @gitaly_repo, + remote_url: remote_url, + http_authorization_header: authorization) response = GitalyClient.call(@storage, :remote_service, :find_remote_root_ref, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index d2dbd456180..6a75096ff80 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -70,13 +70,21 @@ module Gitlab end.join end - def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:, prune: true, check_tags_changed: false) + # rubocop: disable Metrics/ParameterLists + # The `remote` parameter is going away soonish anyway, at which point the + # Rubocop warning can be enabled again. + def fetch_remote(remote, url:, refmap:, ssh_auth:, forced:, no_tags:, timeout:, prune: true, check_tags_changed: false) request = Gitaly::FetchRemoteRequest.new( repository: @gitaly_repo, remote: remote, force: forced, no_tags: no_tags, timeout: timeout, no_prune: !prune, check_tags_changed: check_tags_changed ) + if url + request.remote_params = Gitaly::Remote.new(url: url, + mirror_refmaps: Array.wrap(refmap).map(&:to_s)) + end + if ssh_auth&.ssh_mirror_url? if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? request.ssh_key = ssh_auth.ssh_private_key @@ -89,6 +97,7 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :fetch_remote, request, timeout: GitalyClient.long_timeout) end + # rubocop: enable Metrics/ParameterLists def create_repository request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo) diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 7f1569f592f..28cd3f802a2 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -36,7 +36,11 @@ module Gitlab # updating the timestamp. project.update_column(:last_repository_updated_at, Time.zone.now) - project.repository.fetch_remote('github', forced: false) + if Feature.enabled?(:fetch_remote_params, project, default_enabled: :yaml) + project.repository.fetch_remote('github', url: project.import_url, refmap: Gitlab::GithubImport.refmap, forced: false) + else + project.repository.fetch_remote('github', forced: false) + end pname = project.path_with_namespace diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb index 827027203ff..809a518d13a 100644 --- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb @@ -6,6 +6,13 @@ module Gitlab class PullRequestsReviewsImporter include ParallelScheduling + def initialize(...) + super + + @merge_requests_already_imported_cache_key = + "github-importer/merge_request/already-imported/#{project.id}" + end + def importer_class PullRequestReviewImporter end @@ -22,11 +29,31 @@ module Gitlab :pull_request_reviews end - def id_for_already_imported_cache(merge_request) - merge_request.id + def id_for_already_imported_cache(review) + review.id + end + + def each_object_to_import(&block) + if use_github_review_importer_query_only_unimported_merge_requests? + each_merge_request_to_import(&block) + else + each_merge_request_skipping_imported(&block) + end end - def each_object_to_import + private + + attr_reader :merge_requests_already_imported_cache_key + + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62036#note_587181108 + def use_github_review_importer_query_only_unimported_merge_requests? + Feature.enabled?( + :github_review_importer_query_only_unimported_merge_requests, + default_enabled: :yaml + ) + end + + def each_merge_request_skipping_imported project.merge_requests.find_each do |merge_request| next if already_imported?(merge_request) @@ -40,6 +67,67 @@ module Gitlab mark_as_imported(merge_request) end end + + # The worker can be interrupted, by rate limit for instance, + # in different situations. To avoid requesting already imported data, + # if the worker is interrupted: + # - before importing all reviews of a merge request + # The reviews page is cached with the `PageCounter`, by merge request. + # - before importing all merge requests reviews + # Merge requests that had all the reviews imported are cached with + # `mark_merge_request_reviews_imported` + def each_merge_request_to_import + each_review_page do |page, merge_request| + page.objects.each do |review| + next if already_imported?(review) + + review.merge_request_id = merge_request.id + yield(review) + + mark_as_imported(review) + end + end + end + + def each_review_page + merge_requests_to_import.find_each do |merge_request| + # The page counter needs to be scoped by merge request to avoid skipping + # pages of reviews from already imported merge requests. + page_counter = PageCounter.new(project, page_counter_id(merge_request)) + repo = project.import_source + options = collection_options.merge(page: page_counter.current) + + client.each_page(collection_method, repo, merge_request.iid, options) do |page| + next unless page_counter.set(page.number) + + yield(page, merge_request) + end + + # Avoid unnecessary Redis cache keys after the work is done. + page_counter.expire! + mark_merge_request_reviews_imported(merge_request) + end + end + + # Returns only the merge requests that still have reviews to be imported. + def merge_requests_to_import + project.merge_requests.where.not(id: already_imported_merge_requests) # rubocop: disable CodeReuse/ActiveRecord + end + + def already_imported_merge_requests + Gitlab::Cache::Import::Caching.values_from_set(merge_requests_already_imported_cache_key) + end + + def page_counter_id(merge_request) + "merge_request/#{merge_request.id}/#{collection_method}" + end + + def mark_merge_request_reviews_imported(merge_request) + Gitlab::Cache::Import::Caching.set_add( + merge_requests_already_imported_cache_key, + merge_request.id + ) + end end end end diff --git a/lib/gitlab/github_import/page_counter.rb b/lib/gitlab/github_import/page_counter.rb index 3b4fd42ba2a..3face4c794b 100644 --- a/lib/gitlab/github_import/page_counter.rb +++ b/lib/gitlab/github_import/page_counter.rb @@ -26,6 +26,10 @@ module Gitlab def current Gitlab::Cache::Import::Caching.read_integer(cache_key) || 1 end + + def expire! + Gitlab::Cache::Import::Caching.expire(cache_key, 0) + end end end end diff --git a/lib/gitlab/global_id/deprecations.rb b/lib/gitlab/global_id/deprecations.rb new file mode 100644 index 00000000000..ac4a44e0e10 --- /dev/null +++ b/lib/gitlab/global_id/deprecations.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module GlobalId + module Deprecations + Deprecation = Struct.new(:old_model_name, :new_model_name, :milestone, keyword_init: true) + + # Contains the deprecations in place. + # Example: + # + # DEPRECATIONS = [ + # Deprecation.new(old_model_name: 'PrometheusService', new_model_name: 'Integrations::Prometheus', milestone: '14.0') + # ].freeze + DEPRECATIONS = [ + # This works around an accidentally released argument named as `"EEIterationID"` in 7000489db. + Deprecation.new(old_model_name: 'EEIteration', new_model_name: 'Iteration', milestone: '13.3') + ].freeze + + # Maps of the DEPRECATIONS Hash for quick access. + OLD_NAME_MAP = DEPRECATIONS.index_by(&:old_model_name).freeze + NEW_NAME_MAP = DEPRECATIONS.index_by(&:new_model_name).freeze + OLD_GRAPHQL_NAME_MAP = DEPRECATIONS.index_by do |d| + Types::GlobalIDType.model_name_to_graphql_name(d.old_model_name) + end.freeze + + def self.deprecated?(old_model_name) + OLD_NAME_MAP.key?(old_model_name) + end + + def self.deprecation_for(old_model_name) + OLD_NAME_MAP[old_model_name] + end + + def self.deprecation_by(new_model_name) + NEW_NAME_MAP[new_model_name] + end + + # Returns the new `graphql_name` (Type#graphql_name) of a deprecated GID, + # or the `graphql_name` argument given if no deprecation applies. + def self.apply_to_graphql_name(graphql_name) + return graphql_name unless deprecation = OLD_GRAPHQL_NAME_MAP[graphql_name] + + Types::GlobalIDType.model_name_to_graphql_name(deprecation.new_model_name) + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 1fd210c521e..14f9c7f2191 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -47,6 +47,7 @@ module Gitlab push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) + push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql.rb b/lib/gitlab/graphql.rb deleted file mode 100644 index 74c04e5380e..00000000000 --- a/lib/gitlab/graphql.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - StandardGraphqlError = Class.new(StandardError) - end -end diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index 4d575b964e5..dc49c806398 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -51,14 +51,11 @@ module Gitlab object end - # authorizes the object using the current class authorization. def authorize!(object) raise_resource_not_available_error! unless authorized_resource?(object) end def authorized_resource?(object) - # Sanity check. We don't want to accidentally allow a developer to authorize - # without first adding permissions to authorize against raise ConfigurationError, "#{self.class.name} has no authorizations" if self.class.authorization.none? self.class.authorization.ok?(object, current_user) diff --git a/lib/gitlab/graphql/deprecation.rb b/lib/gitlab/graphql/deprecation.rb index 8b73eeb4e52..20068758502 100644 --- a/lib/gitlab/graphql/deprecation.rb +++ b/lib/gitlab/graphql/deprecation.rb @@ -41,7 +41,7 @@ module Gitlab parts = [ "#{deprecated_in(format: :markdown)}.", reason_text, - replacement.then { |r| "Use: [`#{r}`](##{r.downcase.tr('.', '')})." if r } + replacement_markdown.then { |r| "Use: #{r}." if r } ].compact case context @@ -52,6 +52,13 @@ module Gitlab end end + def replacement_markdown + return unless replacement.present? + return "`#{replacement}`" unless replacement.include?('.') # only fully qualified references can be linked + + "[`#{replacement}`](##{replacement.downcase.tr('.', '')})" + end + def edit_description(original_description) @original_description = original_description return unless original_description diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb deleted file mode 100644 index b598b605141..00000000000 --- a/lib/gitlab/graphql/docs/helper.rb +++ /dev/null @@ -1,434 +0,0 @@ -# frozen_string_literal: true - -return if Rails.env.production? - -module Gitlab - module Graphql - module Docs - # We assume a few things about the schema. We use the graphql-ruby gem, which enforces: - # - All mutations have a single input field named 'input' - # - All mutations have a payload type, named after themselves - # - All mutations have an input type, named after themselves - # If these things change, then some of this code will break. Such places - # are guarded with an assertion that our assumptions are not violated. - ViolatedAssumption = Class.new(StandardError) - - SUGGESTED_ACTION = <<~MSG - We expect it to be impossible to violate our assumptions about - how mutation arguments work. - - If that is not the case, then something has probably changed in the - way we generate our schema, perhaps in the library we use: graphql-ruby - - Please ask for help in the #f_graphql or #backend channels. - MSG - - CONNECTION_ARGS = %w[after before first last].to_set - - FIELD_HEADER = <<~MD - #### Fields - - | Name | Type | Description | - | ---- | ---- | ----------- | - MD - - ARG_HEADER = <<~MD - # Arguments - - | Name | Type | Description | - | ---- | ---- | ----------- | - MD - - CONNECTION_NOTE = <<~MD - This field returns a [connection](#connections). It accepts the - four standard [pagination arguments](#connection-pagination-arguments): - `before: String`, `after: String`, `first: Int`, `last: Int`. - MD - - # Helper with functions to be used by HAML templates - # This includes graphql-docs gem helpers class. - # You can check the included module on: https://github.com/gjtorikian/graphql-docs/blob/v1.6.0/lib/graphql-docs/helpers.rb - module Helper - include GraphQLDocs::Helpers - include Gitlab::Utils::StrongMemoize - - def auto_generated_comment - <<-MD.strip_heredoc - --- - stage: Plan - group: Project Management - info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers - --- - - <!--- - This documentation is auto generated by a script. - - Please do not edit this file directly, check compile_docs task on lib/tasks/gitlab/graphql.rake. - ---> - MD - end - - # Template methods: - # Methods that return chunks of Markdown for insertion into the document - - def render_full_field(field, heading_level: 3, owner: nil) - conn = connection?(field) - args = field[:arguments].reject { |arg| conn && CONNECTION_ARGS.include?(arg[:name]) } - arg_owner = [owner, field[:name]] - - chunks = [ - render_name_and_description(field, level: heading_level, owner: owner), - render_return_type(field), - render_input_type(field), - render_connection_note(field), - render_argument_table(heading_level, args, arg_owner), - render_return_fields(field, owner: owner) - ] - - join(:block, chunks) - end - - def render_argument_table(level, args, owner) - arg_header = ('#' * level) + ARG_HEADER - render_field_table(arg_header, args, owner) - end - - def render_name_and_description(object, owner: nil, level: 3) - content = [] - - heading = '#' * level - name = [owner, object[:name]].compact.join('.') - - content << "#{heading} `#{name}`" - content << render_description(object, owner, :block) - - join(:block, content) - end - - def render_object_fields(fields, owner:, level_bump: 0) - return if fields.blank? - - (with_args, no_args) = fields.partition { |f| args?(f) } - type_name = owner[:name] if owner - header_prefix = '#' * level_bump - sections = [ - render_simple_fields(no_args, type_name, header_prefix), - render_fields_with_arguments(with_args, type_name, header_prefix) - ] - - join(:block, sections) - end - - def render_enum_value(enum, value) - render_row(render_name(value, enum[:name]), render_description(value, enum[:name], :inline)) - end - - def render_union_member(member) - "- [`#{member}`](##{member.downcase})" - end - - # QUERIES: - - # Methods that return parts of the schema, or related information: - - def connection_object_types - objects.select { |t| t[:is_edge] || t[:is_connection] } - end - - def object_types - objects.reject { |t| t[:is_edge] || t[:is_connection] || t[:is_payload] } - end - - def interfaces - graphql_interface_types.map { |t| t.merge(fields: t[:fields] + t[:connections]) } - end - - def fields_of(type_name) - graphql_operation_types - .find { |type| type[:name] == type_name } - .values_at(:fields, :connections) - .flatten - .then { |fields| sorted_by_name(fields) } - end - - # Place the arguments of the input types on the mutation itself. - # see: `#input_types` - this method must not call `#input_types` to avoid mutual recursion - def mutations - @mutations ||= sorted_by_name(graphql_mutation_types).map do |t| - inputs = t[:input_fields] - input = inputs.first - name = t[:name] - - assert!(inputs.one?, "Expected exactly 1 input field named #{name}. Found #{inputs.count} instead.") - assert!(input[:name] == 'input', "Expected the input of #{name} to be named 'input'") - - input_type_name = input[:type][:name] - input_type = graphql_input_object_types.find { |t| t[:name] == input_type_name } - assert!(input_type.present?, "Cannot find #{input_type_name} for #{name}.input") - - arguments = input_type[:input_fields] - seen_type!(input_type_name) - t.merge(arguments: arguments) - end - end - - # We assume that the mutations have been processed first, marking their - # inputs as `seen_type?` - def input_types - mutations # ensure that mutations have seen their inputs first - graphql_input_object_types.reject { |t| seen_type?(t[:name]) } - end - - # We ignore the built-in enum types, and sort values by name - def enums - graphql_enum_types - .reject { |type| type[:values].empty? } - .reject { |enum_type| enum_type[:name].start_with?('__') } - .map { |type| type.merge(values: sorted_by_name(type[:values])) } - end - - private # DO NOT CALL THESE METHODS IN TEMPLATES - - # Template methods - - def render_return_type(query) - return unless query[:type] # for example, mutations - - "Returns #{render_field_type(query[:type])}." - end - - def render_simple_fields(fields, type_name, header_prefix) - render_field_table(header_prefix + FIELD_HEADER, fields, type_name) - end - - def render_fields_with_arguments(fields, type_name, header_prefix) - return if fields.empty? - - level = 5 + header_prefix.length - sections = sorted_by_name(fields).map do |f| - render_full_field(f, heading_level: level, owner: type_name) - end - - <<~MD.chomp - #{header_prefix}#### Fields with arguments - - #{join(:block, sections)} - MD - end - - def render_field_table(header, fields, owner) - return if fields.empty? - - fields = sorted_by_name(fields) - header + join(:table, fields.map { |f| render_field(f, owner) }) - end - - def render_field(field, owner) - render_row( - render_name(field, owner), - render_field_type(field[:type]), - render_description(field, owner, :inline) - ) - end - - def render_return_fields(mutation, owner:) - fields = mutation[:return_fields] - return if fields.blank? - - name = owner.to_s + mutation[:name] - render_object_fields(fields, owner: { name: name }) - end - - def render_connection_note(field) - return unless connection?(field) - - CONNECTION_NOTE.chomp - end - - def render_row(*values) - "| #{values.map { |val| val.to_s.squish }.join(' | ')} |" - end - - def render_name(object, owner = nil) - rendered_name = "`#{object[:name]}`" - rendered_name += ' **{warning-solid}**' if deprecated?(object, owner) - - return rendered_name unless owner - - owner = Array.wrap(owner).join('') - id = (owner + object[:name]).downcase - - %(<a id="#{id}"></a>) + rendered_name - end - - # Returns the object description. If the object has been deprecated, - # the deprecation reason will be returned in place of the description. - def render_description(object, owner = nil, context = :block) - if deprecated?(object, owner) - render_deprecation(object, owner, context) - else - render_description_of(object, owner, context) - end - end - - def deprecated?(object, owner) - return true if object[:is_deprecated] # only populated for fields, not arguments! - - key = [*Array.wrap(owner), object[:name]].join('.') - deprecations.key?(key) - end - - def render_description_of(object, owner, context = nil) - desc = if object[:is_edge] - base = object[:name].chomp('Edge') - "The edge type for [`#{base}`](##{base.downcase})." - elsif object[:is_connection] - base = object[:name].chomp('Connection') - "The connection type for [`#{base}`](##{base.downcase})." - else - object[:description]&.strip - end - - return if desc.blank? - - desc += '.' unless desc.ends_with?('.') - see = doc_reference(object, owner) - desc += " #{see}" if see - desc += " (see [Connections](#connections))" if connection?(object) && context != :block - desc - end - - def doc_reference(object, owner) - field = schema_field(owner, object[:name]) if owner - return unless field - - ref = field.try(:doc_reference) - return if ref.blank? - - parts = ref.to_a.map do |(title, url)| - "[#{title.strip}](#{url.strip})" - end - - "See #{parts.join(', ')}." - end - - def render_deprecation(object, owner, context) - buff = [] - deprecation = schema_deprecation(owner, object[:name]) - - buff << (deprecation&.original_description || render_description_of(object, owner)) if context == :block - buff << if deprecation - deprecation.markdown(context: context) - else - "**Deprecated:** #{object[:deprecation_reason]}" - end - - join(context, buff) - end - - def render_field_type(type) - "[`#{type[:info]}`](##{type[:name].downcase})" - end - - def join(context, chunks) - chunks.compact! - return if chunks.blank? - - case context - when :block - chunks.join("\n\n") - when :inline - chunks.join(" ").squish.presence - when :table - chunks.join("\n") - end - end - - # Queries - - def sorted_by_name(objects) - return [] unless objects.present? - - objects.sort_by { |o| o[:name] } - end - - def connection?(field) - type_name = field.dig(:type, :name) - type_name.present? && type_name.ends_with?('Connection') - end - - # We are ignoring connections and built in types for now, - # they should be added when queries are generated. - def objects - strong_memoize(:objects) do - mutations = schema.mutation&.fields&.keys&.to_set || [] - - graphql_object_types - .reject { |object_type| object_type[:name]["__"] || object_type[:name] == 'Subscription' } # We ignore introspection and subscription types. - .map do |type| - name = type[:name] - type.merge( - is_edge: name.ends_with?('Edge'), - is_connection: name.ends_with?('Connection'), - is_payload: name.ends_with?('Payload') && mutations.include?(name.chomp('Payload').camelcase(:lower)), - fields: type[:fields] + type[:connections] - ) - end - end - end - - def args?(field) - args = field[:arguments] - return false if args.blank? - return true unless connection?(field) - - args.any? { |arg| CONNECTION_ARGS.exclude?(arg[:name]) } - end - - # returns the deprecation information for a field or argument - # See: Gitlab::Graphql::Deprecation - def schema_deprecation(type_name, field_name) - key = [*Array.wrap(type_name), field_name].join('.') - deprecations[key] - end - - def render_input_type(query) - input_field = query[:input_fields]&.first - return unless input_field - - "Input type: `#{input_field[:type][:name]}`" - end - - def schema_field(type_name, field_name) - type = schema.types[type_name] - return unless type && type.kind.fields? - - type.fields[field_name] - end - - def deprecations - strong_memoize(:deprecations) do - mapping = {} - - schema.types.each do |type_name, type| - next unless type.kind.fields? - - type.fields.each do |field_name, field| - mapping["#{type_name}.#{field_name}"] = field.try(:deprecation) - field.arguments.each do |arg_name, arg| - mapping["#{type_name}.#{field_name}.#{arg_name}"] = arg.try(:deprecation) - end - end - end - - mapping.compact - end - end - - def assert!(claim, message) - raise ViolatedAssumption, "#{message}\n#{SUGGESTED_ACTION}" unless claim - end - end - end - end -end diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb deleted file mode 100644 index ae0898e6198..00000000000 --- a/lib/gitlab/graphql/docs/renderer.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -return if Rails.env.production? - -module Gitlab - module Graphql - module Docs - # Gitlab renderer for graphql-docs. - # Uses HAML templates to parse markdown and generate .md files. - # It uses graphql-docs helpers and schema parser, more information in https://github.com/gjtorikian/graphql-docs. - # - # Arguments: - # schema - the GraphQL schema definition. For GitLab should be: GitlabSchema - # output_dir: The folder where the markdown files will be saved - # template: The path of the haml template to be parsed - class Renderer - include Gitlab::Graphql::Docs::Helper - - attr_reader :schema - - def initialize(schema, output_dir:, template:) - @output_dir = output_dir - @template = template - @layout = Haml::Engine.new(File.read(template)) - @parsed_schema = GraphQLDocs::Parser.new(schema.graphql_definition, {}).parse - @schema = schema - @seen = Set.new - end - - def contents - # Render and remove an extra trailing new line - @contents ||= @layout.render(self).sub!(/\n(?=\Z)/, '') - end - - def write - filename = File.join(@output_dir, 'index.md') - - FileUtils.mkdir_p(@output_dir) - File.write(filename, contents) - end - - private - - def seen_type?(name) - @seen.include?(name) - end - - def seen_type!(name) - @seen << name - end - end - end - end -end diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml deleted file mode 100644 index 7d42fb3a9f8..00000000000 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ /dev/null @@ -1,224 +0,0 @@ --# haml-lint:disable UnnecessaryStringOutput - -= auto_generated_comment - -:plain - # GraphQL API Resources - - This documentation is self-generated based on GitLab current GraphQL schema. - - The API can be explored interactively using the [GraphiQL IDE](../index.md#graphiql). - - Each table below documents a GraphQL type. Types match loosely to models, but not all - fields and methods on a model are available via GraphQL. - - WARNING: - Fields that are deprecated are marked with **{warning-solid}**. - Items (fields, enums, etc) that have been removed according to our [deprecation process](../index.md#deprecation-and-removal-process) can be found - in [Removed Items](../removed_items.md). - - <!-- vale off --> - <!-- Docs linting disabled after this line. --> - <!-- See https://docs.gitlab.com/ee/development/documentation/testing.html#disable-vale-tests --> -\ - -:plain - ## `Query` type - - The `Query` type contains the API's top-level entry points for all executable queries. -\ - -- fields_of('Query').each do |field| - = render_full_field(field, heading_level: 3, owner: 'Query') - \ - -:plain - ## `Mutation` type - - The `Mutation` type contains all the mutations you can execute. - - All mutations receive their arguments in a single input object named `input`, and all mutations - support at least a return field `errors` containing a list of error messages. - - All input objects may have a `clientMutationId: String` field, identifying the mutation. - - For example: - - ```graphql - mutation($id: NoteableID!, $body: String!) { - createNote(input: { noteableId: $id, body: $body }) { - errors - } - } - ``` -\ - -- mutations.each do |field| - = render_full_field(field, heading_level: 3, owner: 'Mutation') - \ - -:plain - ## Connections - - Some types in our schema are `Connection` types - they represent a paginated - collection of edges between two nodes in the graph. These follow the - [Relay cursor connections specification](https://relay.dev/graphql/connections.htm). - - ### Pagination arguments {#connection-pagination-arguments} - - All connection fields support the following pagination arguments: - - | Name | Type | Description | - |------|------|-------------| - | `after` | [`String`](#string) | Returns the elements in the list that come after the specified cursor. | - | `before` | [`String`](#string) | Returns the elements in the list that come before the specified cursor. | - | `first` | [`Int`](#int) | Returns the first _n_ elements from the list. | - | `last` | [`Int`](#int) | Returns the last _n_ elements from the list. | - - Since these arguments are common to all connection fields, they are not repeated for each connection. - - ### Connection fields - - All connections have at least the following fields: - - | Name | Type | Description | - |------|------|-------------| - | `pageInfo` | [`PageInfo!`](#pageinfo) | Pagination information. | - | `edges` | `[edge!]` | The edges. | - | `nodes` | `[item!]` | The items in the current page. | - - The precise type of `Edge` and `Item` depends on the kind of connection. A - [`ProjectConnection`](#projectconnection) will have nodes that have the type - [`[Project!]`](#project), and edges that have the type [`ProjectEdge`](#projectedge). - - ### Connection types - - Some of the types in the schema exist solely to model connections. Each connection - has a distinct, named type, with a distinct named edge type. These are listed separately - below. -\ - -- connection_object_types.each do |type| - = render_name_and_description(type, level: 4) - \ - = render_object_fields(type[:fields], owner: type, level_bump: 1) - \ - -:plain - ## Object types - - Object types represent the resources that the GitLab GraphQL API can return. - They contain _fields_. Each field has its own type, which will either be one of the - basic GraphQL [scalar types](https://graphql.org/learn/schema/#scalar-types) - (e.g.: `String` or `Boolean`) or other object types. Fields may have arguments. - Fields with arguments are exactly like top-level queries, and are listed beneath - the table of fields for each object type. - - For more information, see - [Object Types and Fields](https://graphql.org/learn/schema/#object-types-and-fields) - on `graphql.org`. -\ - -- object_types.each do |type| - = render_name_and_description(type) - \ - = render_object_fields(type[:fields], owner: type) - \ - -:plain - ## Enumeration types - - Also called _Enums_, enumeration types are a special kind of scalar that - is restricted to a particular set of allowed values. - - For more information, see - [Enumeration Types](https://graphql.org/learn/schema/#enumeration-types) - on `graphql.org`. -\ - -- enums.each do |enum| - = render_name_and_description(enum) - \ - ~ "| Value | Description |" - ~ "| ----- | ----------- |" - - enum[:values].each do |value| - = render_enum_value(enum, value) - \ - -:plain - ## Scalar types - - Scalar values are atomic values, and do not have fields of their own. - Basic scalars include strings, boolean values, and numbers. This schema also - defines various custom scalar values, such as types for times and dates. - - This schema includes custom scalar types for identifiers, with a specific type for - each kind of object. - - For more information, read about [Scalar Types](https://graphql.org/learn/schema/#scalar-types) on `graphql.org`. -\ - -- graphql_scalar_types.each do |type| - = render_name_and_description(type) - \ - -:plain - ## Abstract types - - Abstract types (unions and interfaces) are ways the schema can represent - values that may be one of several concrete types. - - - A [`Union`](https://graphql.org/learn/schema/#union-types) is a set of possible types. - The types might not have any fields in common. - - An [`Interface`](https://graphql.org/learn/schema/#interfaces) is a defined set of fields. - Types may `implement` an interface, which - guarantees that they have all the fields in the set. A type may implement more than - one interface. - - See the [GraphQL documentation](https://graphql.org/learn/) for more information on using - abstract types. -\ - -:plain - ### Unions -\ - -- graphql_union_types.each do |type| - = render_name_and_description(type, level: 4) - \ - One of: - \ - - type[:possible_types].each do |member| - = render_union_member(member) - \ - -:plain - ### Interfaces -\ - -- interfaces.each do |type| - = render_name_and_description(type, level: 4) - \ - Implementations: - \ - - type[:implemented_by].each do |type_name| - ~ "- [`#{type_name}`](##{type_name.downcase})" - \ - = render_object_fields(type[:fields], owner: type, level_bump: 1) - \ - -:plain - ## Input types - - Types that may be used as arguments (all scalar types may also - be used as arguments). - - Only general use input types are listed here. For mutation input types, - see the associated mutation type above. -\ - -- input_types.each do |type| - = render_name_and_description(type) - \ - = render_argument_table(3, type[:input_fields], type[:name]) - \ diff --git a/lib/gitlab/graphql/standard_graphql_error.rb b/lib/gitlab/graphql/standard_graphql_error.rb new file mode 100644 index 00000000000..8364c232af2 --- /dev/null +++ b/lib/gitlab/graphql/standard_graphql_error.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# rubocop:disable Cop/CustomErrorClass + +module Gitlab + module Graphql + class StandardGraphqlError < StandardError + end + end +end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb index f7e46fce134..44b85bf886e 100644 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -20,7 +20,8 @@ module Gitlab def check ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && - ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up && + ::Gitlab::HealthChecks::Redis::TraceChunksCheck.check_up end end end diff --git a/lib/gitlab/health_checks/redis/trace_chunks_check.rb b/lib/gitlab/health_checks/redis/trace_chunks_check.rb new file mode 100644 index 00000000000..cf9fa700b0a --- /dev/null +++ b/lib/gitlab/health_checks/redis/trace_chunks_check.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + module Redis + class TraceChunksCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_trace_chunks_ping' + end + + def successful?(result) + result == 'PONG' + end + + # rubocop: disable CodeReuse/ActiveRecord + def check + catch_timeout 10.seconds do + Gitlab::Redis::TraceChunks.with(&:ping) + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/health_checks/unicorn_check.rb b/lib/gitlab/health_checks/unicorn_check.rb deleted file mode 100644 index f0c6fdab600..00000000000 --- a/lib/gitlab/health_checks/unicorn_check.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - # This check can only be run on Unicorn `master` process - class UnicornCheck - extend SimpleAbstractCheck - - class << self - include Gitlab::Utils::StrongMemoize - - private - - def metric_prefix - 'unicorn_check' - end - - def successful?(result) - result > 0 - end - - def check - return unless http_servers - - http_servers.sum(&:worker_processes) - end - - # Traversal of ObjectSpace is expensive, on fully loaded application - # it takes around 80ms. The instances of HttpServers are not a subject - # to change so we can cache the list of servers. - def http_servers - strong_memoize(:http_servers) do - next unless Gitlab::Runtime.unicorn? - - ObjectSpace.each_object(::Unicorn::HttpServer).to_a - end - end - end - end - end -end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index e4857280969..d05ced00a6b 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -11,9 +11,11 @@ module Gitlab end def self.too_large?(size) - return false unless size.to_i > Gitlab.config.extra['maximum_text_highlight_size_kilobytes'] + file_size_limit = Gitlab.config.extra['maximum_text_highlight_size_kilobytes'] - over_highlight_size_limit.increment(source: "text highlighter") if Feature.enabled?(:track_file_size_over_highlight_limit) + return false unless size.to_i > file_size_limit + + over_highlight_size_limit.increment(source: "file size: #{file_size_limit}") if Feature.enabled?(:track_file_size_over_highlight_limit) true end @@ -68,6 +70,8 @@ module Gitlab end def highlight_rich(text, continue: true) + add_highlight_attempt_metric + tag = lexer.tag tokens = lexer.lex(text, continue: continue) Timeout.timeout(timeout_time) { @formatter.format(tokens, context.merge(tag: tag)).html_safe } @@ -88,12 +92,25 @@ module Gitlab Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) end + def add_highlight_attempt_metric + return unless Feature.enabled?(:track_highlight_timeouts) + + highlighting_attempt.increment(source: (@language || "undefined")) + end + def add_highlight_timeout_metric return unless Feature.enabled?(:track_highlight_timeouts) highlight_timeout.increment(source: Gitlab::Runtime.sidekiq? ? "background" : "foreground") end + def highlighting_attempt + @highlight_attempt ||= Gitlab::Metrics.counter( + :file_highlighting_attempt, + 'Counts the times highlighting has been attempted on a file' + ) + end + def highlight_timeout @highlight_timeout ||= Gitlab::Metrics.counter( :highlight_timeout, diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index d5595e80bdf..2d1bb515058 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -7,6 +7,7 @@ module Gitlab assignees labels total_time_spent + time_change ].freeze def self.safe_hook_attributes @@ -43,7 +44,9 @@ module Gitlab description: absolute_image_urls(issue.description), url: Gitlab::UrlBuilder.build(issue), total_time_spent: issue.total_time_spent, + time_change: issue.time_change, human_total_time_spent: issue.human_total_time_spent, + human_time_change: issue.human_time_change, human_time_estimate: issue.human_time_estimate, assignee_ids: issue.assignee_ids, assignee_id: issue.assignee_ids.first, # This key is deprecated diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index ae2ec424ce5..db807a3c557 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -37,6 +37,7 @@ module Gitlab assignees labels total_time_spent + time_change ].freeze alias_method :merge_request, :object @@ -50,7 +51,9 @@ module Gitlab last_commit: merge_request.diff_head_commit&.hook_attrs, work_in_progress: merge_request.work_in_progress?, total_time_spent: merge_request.total_time_spent, + time_change: merge_request.time_change, human_total_time_spent: merge_request.human_total_time_spent, + human_time_change: merge_request.human_time_change, human_time_estimate: merge_request.human_time_estimate, assignee_ids: merge_request.assignee_ids, assignee_id: merge_request.assignee_ids.first, # This key is deprecated diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 023dbd1c601..30e72b58e21 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -40,24 +40,24 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 1, 'cs_CZ' => 1, - 'de' => 19, + 'de' => 18, 'en' => 100, 'eo' => 1, - 'es' => 41, + 'es' => 40, 'fil_PH' => 1, - 'fr' => 14, + 'fr' => 13, 'gl_ES' => 1, 'id_ID' => 0, 'it' => 2, - 'ja' => 45, - 'ko' => 14, + 'ja' => 44, + 'ko' => 13, 'nl_NL' => 1, - 'pl_PL' => 1, - 'pt_BR' => 22, - 'ru' => 32, + 'pl_PL' => 3, + 'pt_BR' => 21, + 'ru' => 30, 'tr_TR' => 17, - 'uk' => 43, - 'zh_CN' => 72, + 'uk' => 42, + 'zh_CN' => 69, 'zh_HK' => 3, 'zh_TW' => 4 }.freeze diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index 1e8009d29c2..78608a946de 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -32,6 +32,10 @@ module Gitlab end end + def delete_export? + false + end + private def send_file diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 959ece4b903..30cd5ccfbcb 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -69,6 +69,7 @@ module Gitlab # the relation_hash, updating references with new object IDs, mapping users using # the "members_mapper" object, also updating notes if required. def create + return @relation_hash if author_relation? return if invalid_relation? || predefined_relation? setup_base_models @@ -95,6 +96,10 @@ module Gitlab relation_class.try(:predefined_id?, @relation_hash['id']) end + def author_relation? + @relation_name == :author + end + def setup_models raise NotImplementedError end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index ace9d83dc9a..6c0b6de9e85 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -15,8 +15,17 @@ module Gitlab end def gzip(dir:, filename:) + gzip_with_options(dir: dir, filename: filename) + end + + def gunzip(dir:, filename:) + gzip_with_options(dir: dir, filename: filename, options: 'd') + end + + def gzip_with_options(dir:, filename:, options: nil) filepath = File.join(dir, filename) cmd = %W(gzip #{filepath}) + cmd << "-#{options}" if options _, status = Gitlab::Popen.popen(cmd) diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 2baf2c61f7c..febfe00af0b 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -32,7 +32,16 @@ module Gitlab Timeout.timeout(TIMEOUT_LIMIT) do stdin, stdout, stderr, wait_thr = Open3.popen3(command, pgroup: true) stdin.close - pgrp = Process.getpgid(wait_thr[:pid]) + + # When validation is performed on a small archive (e.g. 100 bytes) + # `wait_thr` finishes before we can get process group id. Do not + # raise exception in this scenario. + pgrp = begin + Process.getpgid(wait_thr[:pid]) + rescue Errno::ESRCH + nil + end + status = wait_thr.value if status.success? diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index 4af6b03fe94..af0026b8864 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -15,7 +15,7 @@ module Gitlab end def self.file_compression_error - self.new('File compression failed') + self.new('File compression/decompression failed') end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 4b3258f8caa..5274fcec43e 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -28,9 +28,7 @@ module Gitlab copy_archive wait_for_archived_file do - # Disable archive validation by default - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/235949 - validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size) + validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: :yaml) decompress_archive end rescue StandardError => e diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index aceb4821a06..4786c7a52cc 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -70,11 +70,14 @@ ee: - :award_emoji - events: - :push_event_payload + - label_links: + - :label - notes: - :author - :award_emoji - events: - :push_event_payload + - :system_note_metadata - boards: - :board_assignee - :milestone diff --git a/lib/gitlab/import_export/group/legacy_import_export.yml b/lib/gitlab/import_export/group/legacy_import_export.yml index 19611e1b010..0a6234f9f02 100644 --- a/lib/gitlab/import_export/group/legacy_import_export.yml +++ b/lib/gitlab/import_export/group/legacy_import_export.yml @@ -72,6 +72,8 @@ ee: - :award_emoji - events: - :push_event_payload + - label_links: + - :label - notes: - :author - :award_emoji diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb index 2b95c098b59..8b39362b6bb 100644 --- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb +++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb @@ -55,11 +55,11 @@ module Gitlab def relation_reader strong_memoize(:relation_reader) do if @group_hash.present? - ImportExport::JSON::LegacyReader::Hash.new( + ImportExport::Json::LegacyReader::Hash.new( @group_hash, relation_names: reader.group_relation_names) else - ImportExport::JSON::LegacyReader::File.new( + ImportExport::Json::LegacyReader::File.new( File.join(shared.export_path, 'group.json'), relation_names: reader.group_relation_names) end diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index ea7de4cc896..19d707aaca5 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -118,7 +118,7 @@ module Gitlab def relation_reader strong_memoize(:relation_reader) do - ImportExport::JSON::NdjsonReader.new( + ImportExport::Json::NdjsonReader.new( File.join(shared.export_path, 'tree') ) end diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb index 0f588a55f9d..796b9258e57 100644 --- a/lib/gitlab/import_export/group/tree_saver.rb +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -42,7 +42,7 @@ module Gitlab end def serialize(group) - ImportExport::JSON::StreamingSerializer.new( + ImportExport::Json::StreamingSerializer.new( group, group_tree, json_writer, @@ -64,7 +64,7 @@ module Gitlab end def json_writer - @json_writer ||= ImportExport::JSON::NdjsonWriter.new(@full_path) + @json_writer ||= ImportExport::Json::NdjsonWriter.new(@full_path) end end end diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb index f29c0a44188..97b34088e3e 100644 --- a/lib/gitlab/import_export/json/legacy_reader.rb +++ b/lib/gitlab/import_export/json/legacy_reader.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class LegacyReader class File < LegacyReader include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb index 7be21410d26..e03ab9f7650 100644 --- a/lib/gitlab/import_export/json/legacy_writer.rb +++ b/lib/gitlab/import_export/json/legacy_writer.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class LegacyWriter include Gitlab::ImportExport::CommandLineUtil diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb index 5c8edd485e5..4899bd3b0ee 100644 --- a/lib/gitlab/import_export/json/ndjson_reader.rb +++ b/lib/gitlab/import_export/json/ndjson_reader.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class NdjsonReader MAX_JSON_DOCUMENT_SIZE = 50.megabytes diff --git a/lib/gitlab/import_export/json/ndjson_writer.rb b/lib/gitlab/import_export/json/ndjson_writer.rb index e74fdd74049..e303ac6eefa 100644 --- a/lib/gitlab/import_export/json/ndjson_writer.rb +++ b/lib/gitlab/import_export/json/ndjson_writer.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class NdjsonWriter include Gitlab::ImportExport::CommandLineUtil diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index ec42c5e51c0..d1e013a151c 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class StreamingSerializer include Gitlab::ImportExport::CommandLineUtil diff --git a/lib/gitlab/import_export/legacy_relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb index f8b8b74ffd7..c6b961ea210 100644 --- a/lib/gitlab/import_export/legacy_relation_tree_saver.rb +++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb @@ -22,7 +22,7 @@ module Gitlab private def batch_size(exportable) - Gitlab::ImportExport::JSON::StreamingSerializer.batch_size(exportable) + Gitlab::ImportExport::Json::StreamingSerializer.batch_size(exportable) end end end diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index 113502b4e3c..d8992061524 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -56,13 +56,13 @@ module Gitlab def ndjson_relation_reader return unless Feature.enabled?(:project_import_ndjson, project.namespace, default_enabled: true) - ImportExport::JSON::NdjsonReader.new( + ImportExport::Json::NdjsonReader.new( File.join(shared.export_path, 'tree') ) end def legacy_relation_reader - ImportExport::JSON::LegacyReader::File.new( + ImportExport::Json::LegacyReader::File.new( File.join(shared.export_path, 'project.json'), relation_names: reader.project_relation_names, allowed_path: importable_path diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 16012f3c0c0..1f0fa249390 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -14,7 +14,7 @@ module Gitlab end def save - ImportExport::JSON::StreamingSerializer.new( + ImportExport::Json::StreamingSerializer.new( exportable, reader.project_tree, json_writer, @@ -56,10 +56,10 @@ module Gitlab @json_writer ||= begin if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace, default_enabled: true) full_path = File.join(@shared.export_path, 'tree') - Gitlab::ImportExport::JSON::NdjsonWriter.new(full_path) + Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) else full_path = File.join(@shared.export_path, ImportExport.project_filename) - Gitlab::ImportExport::JSON::LegacyWriter.new(full_path, allowed_path: 'project') + Gitlab::ImportExport::Json::LegacyWriter.new(full_path, allowed_path: 'project') end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index f295ab38de0..5cb1c1f8981 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -88,7 +88,7 @@ module Gitlab when 'Project' @exportable.disk_path when 'Group' - @exportable.full_path + Storage::Hashed.new(@exportable, prefix: Storage::Hashed::GROUP_REPOSITORY_PATH_PREFIX).disk_path else raise Gitlab::ImportExport::Error, "Unsupported Exportable Type #{@exportable&.class}" end diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index d1ac6a55fb7..ab0e56adc32 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -8,8 +8,9 @@ module Gitlab Cache = Class.new(RedisBase).enable_redis_cluster_validation Queues = Class.new(RedisBase) SharedState = Class.new(RedisBase).enable_redis_cluster_validation + TraceChunks = Class.new(RedisBase).enable_redis_cluster_validation - STORAGES = [ActionCable, Cache, Queues, SharedState].freeze + STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks].freeze # Milliseconds represented in seconds (from 1 millisecond to 2 seconds). QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze @@ -21,10 +22,6 @@ module Gitlab nil end - def known_payload_keys - super + STORAGES.flat_map(&:known_payload_keys) - end - def payload super.merge(*STORAGES.flat_map(&:payload)) end diff --git a/lib/gitlab/instrumentation/redis_payload.rb b/lib/gitlab/instrumentation/redis_payload.rb index 69aafffd124..86a6525c8d0 100644 --- a/lib/gitlab/instrumentation/redis_payload.rb +++ b/lib/gitlab/instrumentation/redis_payload.rb @@ -5,12 +5,6 @@ module Gitlab module RedisPayload include ::Gitlab::Utils::StrongMemoize - # Fetches payload keys from the lazy payload (this avoids - # unnecessary processing of the values). - def known_payload_keys - to_lazy_payload.keys - end - def payload to_lazy_payload.transform_values do |value| result = value.call diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb index e6ea98e6d66..9d7254f49f7 100644 --- a/lib/gitlab/integrations/sti_type.rb +++ b/lib/gitlab/integrations/sti_type.rb @@ -4,7 +4,10 @@ module Gitlab module Integrations class StiType < ActiveRecord::Type::String NAMESPACED_INTEGRATIONS = Set.new(%w( - Asana Assembla Bamboo Campfire Confluence Datadog EmailsOnPush + Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog + Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker + Jenkins Jira Mattermost MattermostSlashCommands MicrosoftTeams MockCi Packagist PipelinesEmail Pivotaltracker + Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit Youtrack WebexTeams )).freeze def cast(value) @@ -29,12 +32,16 @@ module Gitlab private + def namespaced_integrations + NAMESPACED_INTEGRATIONS + end + def new_cast(value) value = prepare_value(value) return unless value stripped_name = value.delete_suffix('Service') - return unless NAMESPACED_INTEGRATIONS.include?(stripped_name) + return unless namespaced_integrations.include?(stripped_name) "Integrations::#{stripped_name}" end @@ -55,3 +62,5 @@ module Gitlab end end end + +Gitlab::Integrations::StiType.prepend_mod diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 561cd4509b1..767ce310b5a 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -242,7 +242,7 @@ module Gitlab def self.encode(object, limit: 25.megabytes) return ::Gitlab::Json.dump(object) unless Feature.enabled?(:json_limited_encoder) - buffer = [] + buffer = StringIO.new buffer_size = 0 ::Yajl::Encoder.encode(object) do |data_chunk| @@ -254,7 +254,7 @@ module Gitlab buffer_size += chunk_size end - buffer.join('') + buffer.string end end end diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 7b2c792ebca..a4663314b3b 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -45,6 +45,13 @@ module Gitlab Gitlab.config.gitlab_kas.external_url end + # Return GitLab KAS internal_url + # + # @return [String] internal_url + def internal_url + Gitlab.config.gitlab_kas.internal_url + end + # Return whether GitLab KAS is enabled # # @return [Boolean] external_url diff --git a/lib/gitlab/kas/client.rb b/lib/gitlab/kas/client.rb new file mode 100644 index 00000000000..6675903e692 --- /dev/null +++ b/lib/gitlab/kas/client.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Kas + class Client + TIMEOUT = 2.seconds.freeze + JWT_AUDIENCE = 'gitlab-kas' + + STUB_CLASSES = { + configuration_project: Gitlab::Agent::ConfigurationProject::Rpc::ConfigurationProject::Stub + }.freeze + + ConfigurationError = Class.new(StandardError) + + def initialize + raise ConfigurationError, 'GitLab KAS is not enabled' unless Gitlab::Kas.enabled? + raise ConfigurationError, 'KAS internal URL is not configured' unless Gitlab::Kas.internal_url.present? + end + + def list_agent_config_files(project:) + request = Gitlab::Agent::ConfigurationProject::Rpc::ListAgentConfigFilesRequest.new( + repository: repository(project), + gitaly_address: gitaly_address(project) + ) + + stub_for(:configuration_project) + .list_agent_config_files(request, metadata: metadata) + .config_files + .to_a + end + + private + + def stub_for(service) + @stubs ||= {} + @stubs[service] ||= STUB_CLASSES.fetch(service).new(kas_endpoint_url, credentials, timeout: TIMEOUT) + end + + def repository(project) + gitaly_repository = project.repository.gitaly_repository + + Gitlab::Agent::Modserver::Repository.new(gitaly_repository.to_h) + end + + def gitaly_address(project) + connection_data = Gitlab::GitalyClient.connection_data(project.repository_storage) + + Gitlab::Agent::Modserver::GitalyAddress.new(connection_data) + end + + def kas_endpoint_url + Gitlab::Kas.internal_url.delete_prefix('grpc://') + end + + def credentials + if Rails.env.test? || Rails.env.development? + :this_channel_is_insecure + else + GRPC::Core::ChannelCredentials.new + end + end + + def metadata + { 'authorization' => "bearer #{token}" } + end + + def token + JSONWebToken::HMACToken.new(Gitlab::Kas.secret).tap do |token| + token.issuer = Settings.gitlab.host + token.audience = JWT_AUDIENCE + end.encoded + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/parsers/list_v2.rb b/lib/gitlab/kubernetes/helm/parsers/list_v2.rb deleted file mode 100644 index c5c5d198a6c..00000000000 --- a/lib/gitlab/kubernetes/helm/parsers/list_v2.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - module Parsers - # Parses Helm v2 list (JSON) output - class ListV2 - ParserError = Class.new(StandardError) - - attr_reader :contents, :json - - def initialize(contents) - @contents = contents - @json = Gitlab::Json.parse(contents) - rescue JSON::ParserError => e - raise ParserError, e.message - end - - def releases - @releases = helm_releases - end - - private - - def helm_releases - helm_releases = json['Releases'] || [] - - raise ParserError, 'Invalid format for Releases' unless helm_releases.all? { |item| item.is_a?(Hash) } - - helm_releases - end - end - end - end - end -end diff --git a/lib/gitlab/markdown_cache/field_data.rb b/lib/gitlab/markdown_cache/field_data.rb index 14622c0f186..75364570640 100644 --- a/lib/gitlab/markdown_cache/field_data.rb +++ b/lib/gitlab/markdown_cache/field_data.rb @@ -9,7 +9,7 @@ module Gitlab @data = {} end - delegate :[], :[]=, to: :@data + delegate :[], :[]=, :key?, to: :@data def markdown_fields @data.keys diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 7bd55cce363..4c4942c12d5 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -16,6 +16,10 @@ module Gitlab @error end + def self.record_duration_for_status?(status) + status.to_i.between?(200, 499) + end + # Tracks an event. # # See `Gitlab::Metrics::Transaction#add_event` for more details. diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index 558454eaa1c..756e6b0641a 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -30,8 +30,7 @@ module Gitlab # application: https://gitlab.com/gitlab-org/gitlab/issues/35343 self.readiness_checks = [ WebExporter::ExporterCheck.new(self), - Gitlab::HealthChecks::PumaCheck, - Gitlab::HealthChecks::UnicornCheck + Gitlab::HealthChecks::PumaCheck ] end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 19a835b9fc4..b99261b5c4d 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -15,7 +15,6 @@ module Gitlab HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze - FEATURE_CATEGORY_HEADER = 'X-Gitlab-Feature-Category' FEATURE_CATEGORY_DEFAULT = 'unknown' # These were the top 5 categories at a point in time, chosen as a @@ -67,18 +66,16 @@ module Gitlab def call(env) method = env['REQUEST_METHOD'].downcase method = 'INVALID' unless HTTP_METHODS.key?(method) - started = Time.now.to_f + started = Gitlab::Metrics::System.monotonic_time health_endpoint = health_endpoint?(env['PATH_INFO']) status = 'undefined' - feature_category = nil begin status, headers, body = @app.call(env) - elapsed = Time.now.to_f - started - feature_category = headers&.fetch(FEATURE_CATEGORY_HEADER, nil) + elapsed = Gitlab::Metrics::System.monotonic_time - started - unless health_endpoint + if !health_endpoint && Gitlab::Metrics.record_duration_for_status?(status) RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method }, elapsed) end @@ -104,6 +101,10 @@ module Gitlab HEALTH_ENDPOINT.match?(CGI.unescape(path)) end + + def feature_category + ::Gitlab::ApplicationContext.current_context_attribute(:feature_category) + end end end end diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb index 0a0ac6c5386..5d7f434b660 100644 --- a/lib/gitlab/metrics/samplers/database_sampler.rb +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -45,8 +45,8 @@ module Gitlab def labels_for_class(klass) { - host: klass.connection_config[:host], - port: klass.connection_config[:port], + host: klass.connection_db_config.host, + port: klass.connection_db_config.configuration_hash[:port], class: klass.to_s } end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 3d29d38fa1f..b1c5e9800da 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'prometheus/client/support/unicorn' - module Gitlab module Metrics module Samplers diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb deleted file mode 100644 index 2fa324f3fea..00000000000 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Samplers - class UnicornSampler < BaseSampler - DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 - - def metrics - @metrics ||= init_metrics - end - - def init_metrics - { - unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max), - unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max), - unicorn_workers: ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers') - } - end - - def enabled? - # Raindrops::Linux.tcp_listener_stats is only present on Linux - unicorn_with_listeners? && Raindrops::Linux.respond_to?(:tcp_listener_stats) - end - - def sample - Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| - set_unicorn_connection_metrics('tcp', addr, stats) - end - Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| - set_unicorn_connection_metrics('unix', addr, stats) - end - - metrics[:unicorn_workers].set({}, unicorn_workers_count) - end - - private - - def tcp_listeners - @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) - end - - def set_unicorn_connection_metrics(type, addr, stats) - labels = { socket_type: type, socket_address: addr } - - metrics[:unicorn_active_connections].set(labels, stats.active) - metrics[:unicorn_queued_connections].set(labels, stats.queued) - end - - def unix_listeners - @unix_listeners ||= Unicorn.listener_names - tcp_listeners - end - - def unicorn_with_listeners? - defined?(Unicorn) && Unicorn.listener_names.any? - end - - def unicorn_workers_count - http_servers.sum(&:worker_processes) - end - - # Traversal of ObjectSpace is expensive, on fully loaded application - # it takes around 80ms. The instances of HttpServers are not a subject - # to change so we can cache the list of servers. - def http_servers - return [] unless Gitlab::Runtime.unicorn? - - @http_servers ||= ObjectSpace.each_object(::Unicorn::HttpServer).to_a - end - end - end - end -end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 3db3317e833..9f7884e1364 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -14,6 +14,14 @@ module Gitlab SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze + DB_LOAD_BALANCING_COUNTERS = %i{ + db_replica_count db_replica_cached_count db_replica_wal_count + db_primary_count db_primary_cached_count db_primary_wal_count + }.freeze + DB_LOAD_BALANCING_DURATIONS = %i{db_primary_duration_s db_replica_duration_s}.freeze + + SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze + # This event is published from ActiveRecordBaseTransactionMetrics and # used to record a database transaction duration when calling # ActiveRecord::Base.transaction {} block. @@ -39,23 +47,56 @@ module Gitlab observe(:gitlab_sql_duration_seconds, event) do buckets SQL_DURATION_BUCKET end + + if ::Gitlab::Database::LoadBalancing.enable? + db_role = ::Gitlab::Database::LoadBalancing.db_role_for_connection(payload[:connection]) + return if db_role.blank? + + increment_db_role_counters(db_role, payload) + observe_db_role_duration(db_role, event) + end end def self.db_counter_payload return {} unless Gitlab::SafeRequestStore.active? - payload = {} - DB_COUNTERS.each do |counter| - payload[counter] = Gitlab::SafeRequestStore[counter].to_i + {}.tap do |payload| + DB_COUNTERS.each do |counter| + payload[counter] = Gitlab::SafeRequestStore[counter].to_i + end + + if ::Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + DB_LOAD_BALANCING_COUNTERS.each do |counter| + payload[counter] = ::Gitlab::SafeRequestStore[counter].to_i + end + DB_LOAD_BALANCING_DURATIONS.each do |duration| + payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3) + end + end end - payload end - def self.known_payload_keys - DB_COUNTERS + private + + def wal_command?(payload) + payload[:sql].match(SQL_WAL_LOCATION_REGEX) + end + + def increment_db_role_counters(db_role, payload) + increment("db_#{db_role}_count".to_sym) + increment("db_#{db_role}_cached_count".to_sym) if cached_query?(payload) + increment("db_#{db_role}_wal_count".to_sym) if !cached_query?(payload) && wal_command?(payload) end - private + def observe_db_role_duration(db_role, event) + observe("gitlab_sql_#{db_role}_duration_seconds".to_sym, event) do + buckets ::Gitlab::Metrics::Subscribers::ActiveRecord::SQL_DURATION_BUCKET + end + + duration = event.duration / 1000.0 + duration_key = "db_#{db_role}_duration_s".to_sym + ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration + end def ignored_query?(payload) payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql]) @@ -86,5 +127,3 @@ module Gitlab end end end - -Gitlab::Metrics::Subscribers::ActiveRecord.prepend_mod_with('Gitlab::Metrics::Subscribers::ActiveRecord') diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb index 0df64f2897e..60a1b084345 100644 --- a/lib/gitlab/metrics/subscribers/external_http.rb +++ b/lib/gitlab/metrics/subscribers/external_http.rb @@ -14,8 +14,6 @@ module Gitlab COUNTER = :external_http_count DURATION = :external_http_duration_s - KNOWN_PAYLOAD_KEYS = [COUNTER, DURATION].freeze - def self.detail_store ::Gitlab::SafeRequestStore[DETAIL_STORE] ||= [] end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 3ebafb5c5e4..97cc8bed564 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -13,8 +13,6 @@ module Gitlab THREAD_KEY = :_gitlab_metrics_transaction - SMALL_BUCKETS = [0.1, 0.25, 0.5, 1.0, 2.5, 5.0].freeze - # The series to store events (e.g. Git pushes) in. EVENT_SERIES = 'events' @@ -39,29 +37,10 @@ module Gitlab def initialize @methods = {} - - @started_at = nil - @finished_at = nil - end - - def duration - @finished_at ? (@finished_at - @started_at) : 0.0 end def run - Thread.current[THREAD_KEY] = self - - @started_at = System.monotonic_time - - yield - ensure - @finished_at = System.monotonic_time - - observe(:gitlab_transaction_duration_seconds, duration) do - buckets SMALL_BUCKETS - end - - Thread.current[THREAD_KEY] = nil + raise NotImplementedError end # Tracks a business level event diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index ee9e6f449d3..3ebfcc43b0b 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -6,12 +6,29 @@ module Gitlab CONTROLLER_KEY = 'action_controller.instance' ENDPOINT_KEY = 'api.endpoint' ALLOWED_SUFFIXES = Set.new(%w[json js atom rss xml zip]) + SMALL_BUCKETS = [0.1, 0.25, 0.5, 1.0, 2.5, 5.0].freeze def initialize(env) super() @env = env end + def run + Thread.current[THREAD_KEY] = self + + started_at = System.monotonic_time + + status, _, _ = retval = yield + + finished_at = System.monotonic_time + duration = finished_at - started_at + record_duration_if_needed(status, duration) + + retval + ensure + Thread.current[THREAD_KEY] = nil + end + def labels return @labels if @labels @@ -27,6 +44,14 @@ module Gitlab private + def record_duration_if_needed(status, duration) + return unless Gitlab::Metrics.record_duration_for_status?(status) + + observe(:gitlab_transaction_duration_seconds, duration) do + buckets SMALL_BUCKETS + end + end + def labels_from_controller controller = @env[CONTROLLER_KEY] diff --git a/lib/gitlab/nav/top_nav_menu_item.rb b/lib/gitlab/nav/top_nav_menu_item.rb index ee11f1f4560..4cb38e6bb9b 100644 --- a/lib/gitlab/nav/top_nav_menu_item.rb +++ b/lib/gitlab/nav/top_nav_menu_item.rb @@ -8,17 +8,17 @@ module Gitlab # this is already :/. We could also take a hash and manually check every # entry, but it's much more maintainable to do rely on native Ruby. # rubocop: disable Metrics/ParameterLists - def self.build(id:, title:, active: false, icon: '', href: '', method: nil, view: '', css_class: '', data: {}) + def self.build(id:, title:, active: false, icon: '', href: '', view: '', css_class: nil, data: nil, emoji: nil) { id: id, title: title, active: active, icon: icon, href: href, - method: method, view: view.to_s, css_class: css_class, - data: data + data: data || { qa_selector: 'menu_item_link', qa_title: title }, + emoji: emoji } end # rubocop: enable Metrics/ParameterLists diff --git a/lib/gitlab/nav/top_nav_view_model_builder.rb b/lib/gitlab/nav/top_nav_view_model_builder.rb index 60f5b267071..11ca6a3a3ba 100644 --- a/lib/gitlab/nav/top_nav_view_model_builder.rb +++ b/lib/gitlab/nav/top_nav_view_model_builder.rb @@ -6,9 +6,34 @@ module Gitlab def initialize @menu_builder = ::Gitlab::Nav::TopNavMenuBuilder.new @views = {} + @shortcuts = [] end - delegate :add_primary_menu_item, :add_secondary_menu_item, to: :@menu_builder + # Using delegate hides the stacktrace for some errors, so we choose to be explicit. + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62047#note_579031091 + def add_primary_menu_item(**args) + @menu_builder.add_primary_menu_item(**args) + end + + def add_secondary_menu_item(**args) + @menu_builder.add_secondary_menu_item(**args) + end + + def add_shortcut(**args) + item = ::Gitlab::Nav::TopNavMenuItem.build(**args) + + @shortcuts.push(item) + end + + def add_primary_menu_item_with_shortcut(shortcut_class:, shortcut_href: nil, **args) + add_primary_menu_item(**args) + add_shortcut( + id: "#{args.fetch(:id)}-shortcut", + title: args.fetch(:title), + href: shortcut_href || args.fetch(:href), + css_class: shortcut_class + ) + end def add_view(name, props) @views[name] = props @@ -19,6 +44,7 @@ module Gitlab menu.merge({ views: @views, + shortcuts: @shortcuts, activeTitle: _('Menu') }) end diff --git a/lib/gitlab/pagination/keyset/header_builder.rb b/lib/gitlab/pagination/keyset/header_builder.rb index 69c468207f6..888d93d5fe3 100644 --- a/lib/gitlab/pagination/keyset/header_builder.rb +++ b/lib/gitlab/pagination/keyset/header_builder.rb @@ -13,7 +13,6 @@ module Gitlab def add_next_page_header(query_params) link = next_page_link(page_href(query_params)) - header('Links', link) header('Link', link) end diff --git a/lib/gitlab/pagination/keyset/paginator.rb b/lib/gitlab/pagination/keyset/paginator.rb new file mode 100644 index 00000000000..2ec4472fcd6 --- /dev/null +++ b/lib/gitlab/pagination/keyset/paginator.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class Paginator + include Enumerable + + module Base64CursorConverter + def self.dump(cursor_attributes) + Base64.urlsafe_encode64(Gitlab::Json.dump(cursor_attributes)) + end + + def self.parse(cursor) + Gitlab::Json.parse(Base64.urlsafe_decode64(cursor)).with_indifferent_access + end + end + + FORWARD_DIRECTION = 'n' + BACKWARD_DIRECTION = 'p' + + UnsupportedScopeOrder = Class.new(StandardError) + + # scope - ActiveRecord::Relation object with order by clause + # cursor - Encoded cursor attributes as String. Empty value will requests the first page. + # per_page - Number of items per page. + # cursor_converter - Object that serializes and de-serializes the cursor attributes. Implements dump and parse methods. + # direction_key - Symbol that will be the hash key of the direction within the cursor. (default: _kd => keyset direction) + def initialize(scope:, cursor: nil, per_page: 20, cursor_converter: Base64CursorConverter, direction_key: :_kd) + @keyset_scope = build_scope(scope) + @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(@keyset_scope) + @per_page = per_page + @cursor_converter = cursor_converter + @direction_key = direction_key + @has_another_page = false + @at_last_page = false + @at_first_page = false + @cursor_attributes = decode_cursor_attributes(cursor) + + set_pagination_helper_flags! + end + + # rubocop: disable CodeReuse/ActiveRecord + def records + @records ||= begin + items = if paginate_backward? + reversed_order + .apply_cursor_conditions(keyset_scope, cursor_attributes) + .reorder(reversed_order) + .limit(per_page_plus_one) + .to_a + else + order + .apply_cursor_conditions(keyset_scope, cursor_attributes) + .limit(per_page_plus_one) + .to_a + end + + @has_another_page = items.size == per_page_plus_one + items.pop if @has_another_page + items.reverse! if paginate_backward? + items + end + end + # rubocop: enable CodeReuse/ActiveRecord + + # This and has_previous_page? methods are direction aware. In case we paginate backwards, + # has_next_page? will mean that we have a previous page. + def has_next_page? + records + + if at_last_page? + false + elsif paginate_forward? + @has_another_page + elsif paginate_backward? + true + end + end + + def has_previous_page? + records + + if at_first_page? + false + elsif paginate_backward? + @has_another_page + elsif paginate_forward? + true + end + end + + def cursor_for_next_page + if has_next_page? + data = order.cursor_attributes_for_node(records.last) + data[direction_key] = FORWARD_DIRECTION + cursor_converter.dump(data) + else + nil + end + end + + def cursor_for_previous_page + if has_previous_page? + data = order.cursor_attributes_for_node(records.first) + data[direction_key] = BACKWARD_DIRECTION + cursor_converter.dump(data) + end + end + + def cursor_for_first_page + cursor_converter.dump({ direction_key => FORWARD_DIRECTION }) + end + + def cursor_for_last_page + cursor_converter.dump({ direction_key => BACKWARD_DIRECTION }) + end + + delegate :each, :empty?, :any?, to: :records + + private + + attr_reader :keyset_scope, :order, :per_page, :cursor_converter, :direction_key, :cursor_attributes + + delegate :reversed_order, to: :order + + def at_last_page? + @at_last_page + end + + def at_first_page? + @at_first_page + end + + def per_page_plus_one + per_page + 1 + end + + def decode_cursor_attributes(cursor) + cursor.blank? ? {} : cursor_converter.parse(cursor) + end + + def set_pagination_helper_flags! + @direction = cursor_attributes.delete(direction_key.to_s) + + if cursor_attributes.blank? && @direction.blank? + @at_first_page = true + @direction = FORWARD_DIRECTION + elsif cursor_attributes.blank? + if paginate_forward? + @at_first_page = true + else + @at_last_page = true + end + end + end + + def paginate_backward? + @direction == BACKWARD_DIRECTION + end + + def paginate_forward? + @direction == FORWARD_DIRECTION + end + + def build_scope(scope) + keyset_aware_scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope) + + raise(UnsupportedScopeOrder, 'The order on the scope does not support keyset pagination') unless success + + keyset_aware_scope + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/simple_order_builder.rb b/lib/gitlab/pagination/keyset/simple_order_builder.rb index 5ac5737c3be..76d6bbadaa4 100644 --- a/lib/gitlab/pagination/keyset/simple_order_builder.rb +++ b/lib/gitlab/pagination/keyset/simple_order_builder.rb @@ -26,6 +26,8 @@ module Gitlab def build order = if order_values.empty? primary_key_descending_order + elsif Gitlab::Pagination::Keyset::Order.keyset_aware?(scope) + Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) elsif ordered_by_primary_key? primary_key_order elsif ordered_by_other_column? diff --git a/lib/gitlab/patch/action_dispatch_journey_formatter.rb b/lib/gitlab/patch/action_dispatch_journey_formatter.rb deleted file mode 100644 index 2d3b7bb9923..00000000000 --- a/lib/gitlab/patch/action_dispatch_journey_formatter.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Patch - module ActionDispatchJourneyFormatter - def self.prepended(mod) - mod.alias_method(:old_missing_keys, :missing_keys) - mod.remove_method(:missing_keys) - end - - private - - def missing_keys(route, parts) - missing_keys = nil - tests = route.path.requirements_for_missing_keys_check - route.required_parts.each do |key| - case tests[key] - when nil - unless parts[key] - missing_keys ||= [] - missing_keys << key - end - else - unless tests[key].match?(parts[key]) - missing_keys ||= [] - missing_keys << key - end - end - end - missing_keys - end - end - end -end diff --git a/lib/gitlab/patch/global_id.rb b/lib/gitlab/patch/global_id.rb new file mode 100644 index 00000000000..e99f36c7dca --- /dev/null +++ b/lib/gitlab/patch/global_id.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# To support GlobalID arguments that present a model with its old "deprecated" name +# we alter GlobalID so it will correctly find the record with its new model name. +module Gitlab + module Patch + module GlobalID + def initialize(gid, options = {}) + super + + if deprecation = Gitlab::GlobalId::Deprecations.deprecation_for(model_name) + @new_model_name = deprecation.new_model_name + end + end + + def model_name + new_model_name || super + end + + private + + attr_reader :new_model_name + end + end +end diff --git a/lib/gitlab/patch/hangouts_chat_http_override.rb b/lib/gitlab/patch/hangouts_chat_http_override.rb new file mode 100644 index 00000000000..20dc678e251 --- /dev/null +++ b/lib/gitlab/patch/hangouts_chat_http_override.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Patch + module HangoutsChatHTTPOverride + attr_reader :uri + + # See https://github.com/enzinia/hangouts-chat/blob/6a509f61a56e757f8f417578b393b94423831ff7/lib/hangouts_chat/http.rb + def post(payload) + httparty_response = Gitlab::HTTP.post( + uri, + body: payload.to_json, + headers: { 'Content-Type' => 'application/json' }, + parse: nil # Disables automatic response parsing + ) + httparty_response.response + # The rest of the integration expects a Net::HTTP response + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 8618d2da77c..16a6c470213 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -21,13 +21,11 @@ module Gitlab 500.html 502.html 503.html - abuse_reports admin api apple-touch-icon-precomposed.png apple-touch-icon.png assets - autocomplete dashboard deploy.html explore @@ -38,7 +36,6 @@ module Gitlab health_check help import - invites jwt login oauth @@ -48,7 +45,6 @@ module Gitlab robots.txt s search - sent_notifications sitemap sitemap.xml sitemap.xml.gz diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 42f43f998c4..5c9b029a107 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -170,7 +170,7 @@ module Gitlab def self.print_by_total_time(result, options = {}) default_options = { sort_method: :total_time, filter_by: :total_time } - RubyProf::FlatPrinter.new(result).print(STDOUT, default_options.merge(options)) + RubyProf::FlatPrinter.new(result).print($stdout, default_options.merge(options)) end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 6719dc8362b..e52023c4612 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -43,9 +43,20 @@ module Gitlab end end + # rubocop:disable CodeReuse/ActiveRecord def users - super.where(id: @project.team.members) # rubocop:disable CodeReuse/ActiveRecord + results = super + + if @project.is_a?(Array) + team_members_for_projects = User.joins(:project_authorizations).where(project_authorizations: { project_id: @project }) + results = results.where(id: team_members_for_projects) + else + results = results.where(id: @project.team.members) + end + + results end + # rubocop:enable CodeReuse/ActiveRecord def limited_blobs_count @limited_blobs_count ||= blobs(limit: count_limit).count diff --git a/lib/gitlab/prometheus/adapter.rb b/lib/gitlab/prometheus/adapter.rb index 45438d9bf7c..a977040ef6f 100644 --- a/lib/gitlab/prometheus/adapter.rb +++ b/lib/gitlab/prometheus/adapter.rb @@ -19,9 +19,6 @@ module Gitlab end def cluster_prometheus_adapter - application = cluster&.application_prometheus - return application if application&.available? - integration = cluster&.integration_prometheus integration if integration&.available? end diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index b7d58e05651..b53fdd60606 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -155,6 +155,7 @@ module Gitlab params '<1w 3d 2h 14m>' types Issue, MergeRequest condition do + quick_action_target.supports_time_tracking? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end parse_params do |raw_duration| @@ -177,6 +178,7 @@ module Gitlab params '<time(1h30m | -1h30m)> <date(YYYY-MM-DD)>' types Issue, MergeRequest condition do + quick_action_target.supports_time_tracking? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) end parse_params do |raw_time_date| diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index f3c6315cd6a..47c76e98e5c 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -99,7 +99,7 @@ module Gitlab # Allow it to mark as WIP on MR creation page _or_ through MR notes. (quick_action_target.new_record? || current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target)) end - command :draft, :wip do + command :draft do @updates[:wip_event] = quick_action_target.work_in_progress? ? 'unwip' : 'wip' end diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index 8a432edbd78..e62e1172b65 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -11,12 +11,16 @@ module Gitlab end def cache_key(key) - "#{cache_type}:#{key}:set" + "#{cache_namespace}:#{key}:set" + end + + def new_cache_key(key) + super(key) end def clear_cache!(key) with do |redis| - keys = read(key).map { |value| "#{cache_type}:#{value}" } + keys = read(key).map { |value| "#{cache_namespace}:#{value}" } keys << cache_key(key) redis.pipelined do @@ -24,11 +28,5 @@ module Gitlab end end end - - private - - def cache_type - Gitlab::Redis::Cache::CACHE_NAMESPACE - end end end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index a634f12345a..98b66080b42 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -1,36 +1,16 @@ # frozen_string_literal: true -# please require all dependencies below: -require_relative 'wrapper' unless defined?(::Rails) && ::Rails.root.present? - module Gitlab module Redis class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab' - DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380' - REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE' - - class << self - def default_url - DEFAULT_REDIS_CACHE_URL - end - - def config_file_name - # if ENV set for this class, use it even if it points to a file does not exist - file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] - return file_name unless file_name.nil? - - # otherwise, if config files exists for this class, use it - file_name = config_file_path('redis.cache.yml') - return file_name if File.file?(file_name) - # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent - super - end + private - def instrumentation_class - ::Gitlab::Instrumentation::Redis::Cache - end + def raw_config_hash + config = super + config[:url] = 'redis://localhost:6380' if config[:url].blank? + config end end end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index 42d5167beb3..9e291a73bb6 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -1,37 +1,21 @@ # frozen_string_literal: true -# please require all dependencies below: +# We need this require for MailRoom require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) +require 'active_support/core_ext/object/blank' module Gitlab module Redis class Queues < ::Gitlab::Redis::Wrapper SIDEKIQ_NAMESPACE = 'resque:gitlab' MAILROOM_NAMESPACE = 'mail_room:gitlab' - DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381' - REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE' - class << self - def default_url - DEFAULT_REDIS_QUEUES_URL - end + private - def config_file_name - # if ENV set for this class, use it even if it points to a file does not exist - file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] - return file_name if file_name - - # otherwise, if config files exists for this class, use it - file_name = config_file_path('redis.queues.yml') - return file_name if File.file?(file_name) - - # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent - super - end - - def instrumentation_class - ::Gitlab::Instrumentation::Redis::Queues - end + def raw_config_hash + config = super + config[:url] = 'redis://localhost:6381' if config[:url].blank? + config end end end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 2848c9f0b59..d62516bd287 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -1,8 +1,5 @@ # frozen_string_literal: true -# please require all dependencies below: -require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) - module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper @@ -10,30 +7,13 @@ module Gitlab USER_SESSIONS_NAMESPACE = 'session:user:gitlab' USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab' IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2' - DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382' - REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE' - - class << self - def default_url - DEFAULT_REDIS_SHARED_STATE_URL - end - - def config_file_name - # if ENV set for this class, use it even if it points to a file does not exist - file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] - return file_name if file_name - - # otherwise, if config files exists for this class, use it - file_name = config_file_path('redis.shared_state.yml') - return file_name if File.file?(file_name) - # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent - super - end + private - def instrumentation_class - ::Gitlab::Instrumentation::Redis::SharedState - end + def raw_config_hash + config = super + config[:url] = 'redis://localhost:6382' if config[:url].blank? + config end end end diff --git a/lib/gitlab/redis/trace_chunks.rb b/lib/gitlab/redis/trace_chunks.rb new file mode 100644 index 00000000000..a2e77cb5df5 --- /dev/null +++ b/lib/gitlab/redis/trace_chunks.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class TraceChunks < ::Gitlab::Redis::Wrapper + # The data we store on TraceChunks used to be stored on SharedState. + def self.config_fallback + SharedState + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 94ab67ef08a..bbcc2732e89 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -1,16 +1,16 @@ # frozen_string_literal: true # This file should only be used by sub-classes, not directly by any clients of the sub-classes -# please require all dependencies below: + +# Explicitly load parts of ActiveSupport because MailRoom does not load +# Rails. require 'active_support/core_ext/hash/keys' require 'active_support/core_ext/module/delegation' +require 'active_support/core_ext/string/inflections' module Gitlab module Redis class Wrapper - DEFAULT_REDIS_URL = 'redis://localhost:6379' - REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE' - class << self delegate :params, :url, to: :new @@ -51,33 +51,47 @@ module Gitlab end end - def default_url - DEFAULT_REDIS_URL + def config_file_path(filename) + path = File.join(rails_root, 'config', filename) + return path if File.file?(path) end - # Return the absolute path to a Rails configuration file - # - # We use this instead of `Rails.root` because for certain tasks - # utilizing these classes, `Rails` might not be available. - def config_file_path(filename) - File.expand_path("../../../config/#{filename}", __dir__) + # We need this local implementation of Rails.root because MailRoom + # doesn't load Rails. + def rails_root + File.expand_path('../../..', __dir__) end def config_file_name - # if ENV set for wrapper class, use it even if it points to a file does not exist - file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] - return file_name unless file_name.nil? + [ + # Instance specific config sources: + ENV["GITLAB_REDIS_#{store_name.underscore.upcase}_CONFIG_FILE"], + config_file_path("redis.#{store_name.underscore}.yml"), + + # The current Redis instance may have been split off from another one + # (e.g. TraceChunks was split off from SharedState). There are + # installations out there where the lowest priority config source + # (resque.yml) contains bogus values. In those cases, config_file_name + # should resolve to the instance we originated from (the + # "config_fallback") rather than resque.yml. + config_fallback&.config_file_name, + + # Global config sources: + ENV['GITLAB_REDIS_CONFIG_FILE'], + config_file_path('resque.yml') + ].compact.first + end - # otherwise, if config files exists for wrapper class, use it - file_name = config_file_path('resque.yml') - return file_name if File.file?(file_name) + def store_name + name.demodulize + end - # nil will force use of DEFAULT_REDIS_URL when config file is absent + def config_fallback nil end def instrumentation_class - raise NotImplementedError + "::Gitlab::Instrumentation::Redis::#{store_name}".constantize end end @@ -135,7 +149,7 @@ module Gitlab if config_data config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys else - { url: self.class.default_url } + { url: '' } end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ccb4f6e1097..a31f574fad2 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -138,7 +138,8 @@ module Gitlab end def helm_version_regex - @helm_version_regex ||= %r{#{prefixed_semver_regex}}.freeze + # identical to semver_regex, with optional preceding 'v' + @helm_version_regex ||= Regexp.new("\\Av?#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) end def unbounded_semver_regex diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index f73ac628bce..a20e9845fe6 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -17,6 +17,11 @@ module Gitlab "#{type}:#{namespace}:set" end + # NOTE Remove as part of #331319 + def new_cache_key(type) + super("#{type}:#{namespace}") + end + def write(key, value) full_key = cache_key(key) diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index b0bcea0ca69..f60cac0aff0 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -15,8 +15,7 @@ module Gitlab :rails_runner, :rake, :sidekiq, - :test_suite, - :unicorn + :test_suite ].freeze class << self @@ -36,11 +35,6 @@ module Gitlab !!defined?(::Puma) end - # For unicorn, we need to check for actual server instances to avoid false positives. - def unicorn? - !!(defined?(::Unicorn) && defined?(::Unicorn::HttpServer)) - end - def sidekiq? !!(defined?(::Sidekiq) && Sidekiq.server?) end @@ -66,7 +60,7 @@ module Gitlab end def web_server? - puma? || unicorn? + puma? end def action_cable? diff --git a/lib/gitlab/saas.rb b/lib/gitlab/saas.rb new file mode 100644 index 00000000000..8d9d8415cb1 --- /dev/null +++ b/lib/gitlab/saas.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# This module is used to return various SaaS related configurations +# which may be overridden in other variants of GitLab + +module Gitlab + module Saas + def self.com_url + 'https://gitlab.com' + end + + def self.staging_com_url + 'https://staging.gitlab.com' + end + + def self.subdomain_regex + %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}.freeze + end + + def self.dev_url + 'https://dev.gitlab.org' + end + end +end + +Gitlab::Saas.prepend_mod diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 0f2b7b194c9..30cd63e80c0 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -14,15 +14,21 @@ module Gitlab "#{key}:set" end + # NOTE Remove as part of https://gitlab.com/gitlab-org/gitlab/-/issues/331319 + def new_cache_key(key) + "#{cache_namespace}:#{key}:set" + end + # Returns the number of keys deleted by Redis def expire(*keys) return 0 if keys.empty? with do |redis| - keys = keys.map { |key| cache_key(key) } + keys_to_expire = keys.map { |key| cache_key(key) } + keys_to_expire += keys.map { |key| new_cache_key(key) } # NOTE Remove as part of #331319 Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.unlink(*keys) + redis.unlink(*keys_to_expire) end end end @@ -73,5 +79,9 @@ module Gitlab def with(&blk) Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord end + + def cache_namespace + Gitlab::Redis::Cache::CACHE_NAMESPACE + end end end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 3ac20724403..7ed1958a8d0 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -129,7 +129,7 @@ module Gitlab config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } - config[:bin_dir] = Gitlab.config.gitaly.client_path + config[:bin_dir] = File.join(gitaly_dir, '_build', 'bin') # binaries by default are in `_build/bin` config[:gitlab] = { url: Gitlab.config.gitlab.url } config[:logging] = { dir: Rails.root.join('log').to_s } @@ -153,8 +153,14 @@ module Gitlab second_storage_nodes = [{ storage: 'test_second_storage', address: "unix:#{gitaly_dir}/gitaly2.socket", primary: true, token: 'secret' }] storages = [{ name: 'default', node: nodes }, { name: 'test_second_storage', node: second_storage_nodes }] - failover = { enabled: false } - config = { socket_path: "#{gitaly_dir}/praefect.socket", memory_queue_enabled: true, virtual_storage: storages, failover: failover } + failover = { enabled: false, election_strategy: 'local' } + config = { + i_understand_my_election_strategy_is_unsupported_and_will_be_removed_without_warning: true, + socket_path: "#{gitaly_dir}/praefect.socket", + memory_queue_enabled: true, + virtual_storage: storages, + failover: failover + } config[:token] = 'secret' if Rails.env.test? TomlRB.dump(config) diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb index 9490d543dd1..e20834fa912 100644 --- a/lib/gitlab/sidekiq_cluster/cli.rb +++ b/lib/gitlab/sidekiq_cluster/cli.rb @@ -22,7 +22,7 @@ module Gitlab CommandError = Class.new(StandardError) - def initialize(log_output = STDERR) + def initialize(log_output = $stderr) require_relative '../../../lib/gitlab/sidekiq_logging/json_formatter' # As recommended by https://github.com/mperham/sidekiq/wiki/Advanced-Options#concurrency @@ -47,12 +47,6 @@ module Gitlab option_parser.parse!(argv) - # Remove with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 - if @queue_selector && @experimental_queue_selector - raise CommandError, - 'You cannot specify --queue-selector and --experimental-queue-selector together' - end - worker_metadatas = SidekiqConfig::CliMethods.worker_metadatas(@rails_path) worker_queues = SidekiqConfig::CliMethods.worker_queues(@rails_path) @@ -63,8 +57,7 @@ module Gitlab # as a worker attribute query, and resolve the queues for the # queue group using this query. - # Simplify with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 - if @queue_selector || @experimental_queue_selector + if @queue_selector SidekiqConfig::CliMethods.query_queues(queues_or_query_string, worker_metadatas) else SidekiqConfig::CliMethods.expand_queues(queues_or_query_string.split(','), worker_queues) @@ -194,11 +187,6 @@ module Gitlab @queue_selector = queue_selector end - # Remove with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 - opt.on('--experimental-queue-selector', 'DEPRECATED: use --queue-selector-instead') do |experimental_queue_selector| - @experimental_queue_selector = experimental_queue_selector - end - opt.on('-n', '--negate', 'Run workers for all queues in sidekiq_queues.yml except the given ones') do @negate_queues = true end diff --git a/lib/gitlab/sidekiq_config/worker_router.rb b/lib/gitlab/sidekiq_config/worker_router.rb index 946296a24d3..0670e5521df 100644 --- a/lib/gitlab/sidekiq_config/worker_router.rb +++ b/lib/gitlab/sidekiq_config/worker_router.rb @@ -40,7 +40,7 @@ module Gitlab # queue defined in the input routing rules. The input routing rules, as # described above, is an order-matter array of tuples [query, queue_name]. # - # - The query syntax is the same as the "queue selector" detailedly + # - The query syntax follows "worker matching query" detailedly # denoted in doc/administration/operations/extra_sidekiq_processes.md. # # - The queue_name must be a valid Sidekiq queue name. If the queue name diff --git a/lib/gitlab/sidekiq_logging/logs_jobs.rb b/lib/gitlab/sidekiq_logging/logs_jobs.rb index 6f8cc1c60e9..cfe91b9a266 100644 --- a/lib/gitlab/sidekiq_logging/logs_jobs.rb +++ b/lib/gitlab/sidekiq_logging/logs_jobs.rb @@ -14,6 +14,9 @@ module Gitlab job = job.except('error_backtrace', 'error_class', 'error_message') job['class'] = job.delete('wrapped') if job['wrapped'].present? + job['job_size_bytes'] = Sidekiq.dump_json(job['args']).bytesize + job['args'] = ['[COMPRESSED]'] if ::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor.compressed?(job) + # Add process id params job['pid'] = ::Process.pid diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 87fb36d04e9..32194c4926e 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -55,8 +55,6 @@ module Gitlab scheduling_latency_s = ::Gitlab::InstrumentationHelper.queue_duration_for_job(payload) payload['scheduling_latency_s'] = scheduling_latency_s if scheduling_latency_s - payload['job_size_bytes'] = Sidekiq.dump_json(job).bytesize - payload end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index c5b980769f0..30741f29563 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -9,6 +9,8 @@ module Gitlab # eg: `config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator)` def self.server_configurator(metrics: true, arguments_logger: true, memory_killer: true) lambda do |chain| + # Size limiter should be placed at the top + chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Server chain.add ::Gitlab::SidekiqMiddleware::Monitor chain.add ::Gitlab::SidekiqMiddleware::ServerMetrics if metrics chain.add ::Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger @@ -18,6 +20,7 @@ module Gitlab chain.add ::Gitlab::SidekiqMiddleware::BatchLoader chain.add ::Labkit::Middleware::Sidekiq::Server chain.add ::Gitlab::SidekiqMiddleware::InstrumentationLogger + chain.add ::Gitlab::Database::LoadBalancing::SidekiqServerMiddleware if load_balancing_enabled? chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Server chain.add ::Gitlab::SidekiqVersioning::Middleware chain.add ::Gitlab::SidekiqStatus::ServerMiddleware @@ -39,9 +42,13 @@ module Gitlab # Size limiter should be placed at the bottom, but before the metrics midleware chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Client chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics + chain.add ::Gitlab::Database::LoadBalancing::SidekiqClientMiddleware if load_balancing_enabled? end end + + def self.load_balancing_enabled? + ::Gitlab::Database::LoadBalancing.enable? + end + private_class_method :load_balancing_enabled? end end - -Gitlab::SidekiqMiddleware.singleton_class.prepend_mod_with('Gitlab::SidekiqMiddleware') diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 79ac853ea0c..4cf540ce3b8 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -19,6 +19,7 @@ module Gitlab class DuplicateJob DUPLICATE_KEY_TTL = 6.hours DEFAULT_STRATEGY = :until_executing + STRATEGY_NONE = :none attr_reader :existing_jid @@ -51,6 +52,8 @@ module Gitlab end end + job['idempotency_key'] = idempotency_key + self.existing_jid = read_jid.value end @@ -100,6 +103,7 @@ module Gitlab def strategy return DEFAULT_STRATEGY unless worker_klass return DEFAULT_STRATEGY unless worker_klass.respond_to?(:idempotent?) + return STRATEGY_NONE unless worker_klass.deduplication_enabled? worker_klass.get_deduplicate_strategy end @@ -117,7 +121,7 @@ module Gitlab end def idempotency_key - @idempotency_key ||= "#{namespace}:#{idempotency_hash}" + @idempotency_key ||= job['idempotency_key'] || "#{namespace}:#{idempotency_hash}" end def idempotency_hash @@ -129,6 +133,10 @@ module Gitlab end def idempotency_string + # TODO: dump the argument's JSON using `Sidekiq.dump_json` instead + # this should be done in the next release so all jobs are written + # with their idempotency key. + # see https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1090 "#{worker_class_name}:#{arguments.join('-')}" end end diff --git a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb index b542aa4fe4c..1f0c63c5fff 100644 --- a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb +++ b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb @@ -3,24 +3,6 @@ module Gitlab module SidekiqMiddleware class InstrumentationLogger - def self.keys - @keys ||= [ - :cpu_s, - :gitaly_calls, - :gitaly_duration_s, - :rugged_calls, - :rugged_duration_s, - :elasticsearch_calls, - :elasticsearch_duration_s, - :elasticsearch_timed_out_count, - *::Gitlab::Memory::Instrumentation::KEY_MAPPING.values, - *::Gitlab::Instrumentation::Redis.known_payload_keys, - *::Gitlab::Metrics::Subscribers::ActiveRecord.known_payload_keys, - *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS, - *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS - ] - end - def call(worker, job, queue) ::Gitlab::InstrumentationHelper.init_instrumentation_data @@ -37,7 +19,6 @@ module Gitlab # https://github.com/mperham/sidekiq/blob/53bd529a0c3f901879925b8390353129c465b1f2/lib/sidekiq/processor.rb#L115-L118 job[:instrumentation] = {}.tap do |instrumentation_values| ::Gitlab::InstrumentationHelper.add_instrumentation_data(instrumentation_values) - instrumentation_values.slice!(*self.class.keys) end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 474afffcf93..6d130957f36 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -13,6 +13,10 @@ module Gitlab @metrics = init_metrics @metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) + + if ::Gitlab::Database::LoadBalancing.enable? + @metrics[:sidekiq_load_balancing_count] = ::Gitlab::Metrics.counter(:sidekiq_load_balancing_count, 'Sidekiq jobs with load balancing') + end end def call(worker, job, queue) @@ -69,6 +73,15 @@ module Gitlab @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(instrumentation)) @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(instrumentation)) @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(instrumentation)) + + if ::Gitlab::Database::LoadBalancing.enable? && job[:database_chosen] + load_balancing_labels = { + database_chosen: job[:database_chosen], + data_consistency: job[:data_consistency] + } + + @metrics[:sidekiq_load_balancing_count].increment(labels.merge(load_balancing_labels), 1) + end end end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb b/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb new file mode 100644 index 00000000000..bce295d8ba5 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module SizeLimiter + class Compressor + PayloadDecompressionConflictError = Class.new(StandardError) + PayloadDecompressionError = Class.new(StandardError) + + # Level 5 is a good trade-off between space and time + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1054#note_568129605 + COMPRESS_LEVEL = 5 + ORIGINAL_SIZE_KEY = 'original_job_size_bytes' + COMPRESSED_KEY = 'compressed' + + def self.compressed?(job) + job&.has_key?(COMPRESSED_KEY) + end + + def self.compress(job, job_args) + compressed_args = Base64.strict_encode64(Zlib::Deflate.deflate(job_args, COMPRESS_LEVEL)) + + job[COMPRESSED_KEY] = true + job[ORIGINAL_SIZE_KEY] = job_args.bytesize + job['args'] = [compressed_args] + + compressed_args + end + + def self.decompress(job) + return unless compressed?(job) + + validate_args!(job) + + job.except!(ORIGINAL_SIZE_KEY, COMPRESSED_KEY) + job['args'] = Sidekiq.load_json(Zlib::Inflate.inflate(Base64.strict_decode64(job['args'].first))) + rescue Zlib::Error + raise PayloadDecompressionError, 'Fail to decompress Sidekiq job payload' + end + + def self.validate_args!(job) + if job['args'] && job['args'].length != 1 + exception = PayloadDecompressionConflictError.new('Sidekiq argument list should include 1 argument.\ + This means that there is another a middleware interfering with the job payload.\ + That conflicts with the payload compressor') + ::Gitlab::ErrorTracking.track_and_raise_exception(exception) + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/server.rb b/lib/gitlab/sidekiq_middleware/size_limiter/server.rb new file mode 100644 index 00000000000..70b384c8f28 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/size_limiter/server.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module SizeLimiter + class Server + def call(worker, job, queue) + # This middleware should always decompress jobs regardless of the + # limiter mode or size limit. Otherwise, this could leave compressed + # payloads in queues that are then not able to be processed. + ::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor.decompress(job) + + yield + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb index 2c50c4a2157..d86f1609f14 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb @@ -3,76 +3,103 @@ module Gitlab module SidekiqMiddleware module SizeLimiter - # Validate a Sidekiq job payload limit based on current configuration. + # Handle a Sidekiq job payload limit based on current configuration. # This validator pulls the configuration from the environment variables: - # # - GITLAB_SIDEKIQ_SIZE_LIMITER_MODE: the current mode of the size - # limiter. This must be either `track` or `raise`. - # + # limiter. This must be either `track` or `compress`. + # - GITLAB_SIDEKIQ_SIZE_LIMITER_COMPRESSION_THRESHOLD_BYTES: the + # threshold before the input job payload is compressed. # - GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES: the size limit in bytes. # - # If the size of job payload after serialization exceeds the limit, an - # error is tracked raised adhering to the mode. + # In track mode, if a job payload limit exceeds the size limit, an + # event is sent to Sentry and the job is scheduled like normal. + # + # In compress mode, if a job payload limit exceeds the threshold, it is + # then compressed. If the compressed payload still exceeds the limit, the + # job is discarded, and a ExceedLimitError exception is raised. class Validator def self.validate!(worker_class, job) new(worker_class, job).validate! end DEFAULT_SIZE_LIMIT = 0 + DEFAULT_COMPRESION_THRESHOLD_BYTES = 100_000 # 100kb MODES = [ TRACK_MODE = 'track', - RAISE_MODE = 'raise' + COMPRESS_MODE = 'compress' ].freeze - attr_reader :mode, :size_limit + attr_reader :mode, :size_limit, :compression_threshold def initialize( worker_class, job, mode: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_MODE'], + compression_threshold: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_COMPRESSION_THRESHOLD_BYTES'], size_limit: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES'] ) @worker_class = worker_class @job = job + set_mode(mode) + set_compression_threshold(compression_threshold) + set_size_limit(size_limit) + end + + def validate! + return unless @size_limit > 0 + return if allow_big_payload? + + job_args = compress_if_necessary(::Sidekiq.dump_json(@job['args'])) + return if job_args.bytesize <= @size_limit + + exception = exceed_limit_error(job_args) + if compress_mode? + raise exception + else + track(exception) + end + end + + private + + def set_mode(mode) @mode = (mode || TRACK_MODE).to_s.strip unless MODES.include?(@mode) ::Sidekiq.logger.warn "Invalid Sidekiq size limiter mode: #{@mode}. Fallback to #{TRACK_MODE} mode." @mode = TRACK_MODE end + end + + def set_compression_threshold(compression_threshold) + @compression_threshold = (compression_threshold || DEFAULT_COMPRESION_THRESHOLD_BYTES).to_i + if @compression_threshold <= 0 + ::Sidekiq.logger.warn "Invalid Sidekiq size limiter compression threshold: #{@compression_threshold}" + @compression_threshold = DEFAULT_COMPRESION_THRESHOLD_BYTES + end + end + def set_size_limit(size_limit) @size_limit = (size_limit || DEFAULT_SIZE_LIMIT).to_i if @size_limit < 0 ::Sidekiq.logger.warn "Invalid Sidekiq size limiter limit: #{@size_limit}" end end - def validate! - return unless @size_limit > 0 - - return if allow_big_payload? - return if job_size <= @size_limit - - exception = ExceedLimitError.new(@worker_class, job_size, @size_limit) - # This should belong to Gitlab::ErrorTracking. We'll remove this - # after this epic is done: - # https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/396 - exception.set_backtrace(backtrace) - - if raise_mode? - raise exception - else - track(exception) + def exceed_limit_error(job_args) + ExceedLimitError.new(@worker_class, job_args.bytesize, @size_limit).tap do |exception| + # This should belong to Gitlab::ErrorTracking. We'll remove this + # after this epic is done: + # https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/396 + exception.set_backtrace(backtrace) end end - private + def compress_if_necessary(job_args) + return job_args unless compress_mode? + return job_args if job_args.bytesize < @compression_threshold - def job_size - # This maynot be the optimal solution, but can be acceptable solution - # for now. Internally, Sidekiq calls Sidekiq.dump_json everywhere. - # There is no clean way to intefere to prevent double serialization. - @job_size ||= ::Sidekiq.dump_json(@job).bytesize + ::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor.compress(@job, job_args) end def allow_big_payload? @@ -80,8 +107,8 @@ module Gitlab worker_class.respond_to?(:big_payload?) && worker_class.big_payload? end - def raise_mode? - @mode == RAISE_MODE + def compress_mode? + @mode == COMPRESS_MODE end def track(exception) diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index b8affb42372..d28b5fb509a 100644 --- a/lib/gitlab/slash_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -63,7 +63,7 @@ module Gitlab # Convert Markdown to slacks format def format(string) - Slack::Messenger::Util::LinkFormatter.format(string) + ::Slack::Messenger::Util::LinkFormatter.format(string) end def resource_url diff --git a/lib/gitlab/stack_prof.rb b/lib/gitlab/stack_prof.rb index 4b7d93c91ce..97f52491e9e 100644 --- a/lib/gitlab/stack_prof.rb +++ b/lib/gitlab/stack_prof.rb @@ -118,7 +118,6 @@ module Gitlab # # see also: # * https://github.com/puma/puma/blob/master/docs/signals.md#puma-signals - # * https://github.com/phusion/unicorn/blob/master/SIGNALS # * https://github.com/mperham/sidekiq/wiki/Signals Signal.trap('SIGUSR2') do write.write('.') diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 1ceccc64ec0..227962fc0f7 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -61,7 +61,7 @@ module Gitlab def prompt(message, choices = nil) begin print(message) - answer = STDIN.gets.chomp + answer = $stdin.gets.chomp end while choices.present? && !choices.include?(answer) answer end @@ -70,12 +70,12 @@ module Gitlab # # message - custom message to display before input def prompt_for_password(message = 'Enter password: ') - unless STDIN.tty? + unless $stdin.tty? print(message) - return STDIN.gets.chomp + return $stdin.gets.chomp end - STDIN.getpass(message) + $stdin.getpass(message) end # Runs the given command and matches the output against the given pattern diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index e1ca4b5ff6a..e302865c897 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -5,11 +5,19 @@ module Gitlab class GitlabCiYmlTemplate < BaseTemplate BASE_EXCLUDED_PATTERNS = [%r{\.latest\.}].freeze + TEMPLATES_WITH_LATEST_VERSION = { + 'Jobs/Browser-Performance-Testing' => true, + 'Security/API-Fuzzing' => true, + 'Security/DAST' => true, + 'Terraform' => true + }.freeze + def description "# This file is a template, and might need editing before it works on your project." end class << self + extend ::Gitlab::Utils::Override include Gitlab::Utils::StrongMemoize def extension @@ -54,6 +62,31 @@ module Gitlab excluded_patterns: self.excluded_patterns ) end + + override :find + def find(key, project = nil) + if try_redirect_to_latest?(key, project) + key += '.latest' + end + + super(key, project) + end + + private + + # To gauge the impact of the latest template, + # you can redirect the stable template to the latest template by enabling the feature flag. + # See https://docs.gitlab.com/ee/development/cicd/templates.html#versioning for more information. + def try_redirect_to_latest?(key, project) + return false unless templates_with_latest_version[key] + + flag_name = "redirect_to_latest_template_#{key.underscore.tr('/', '_')}" + ::Feature.enabled?(flag_name, project, default_enabled: :yaml) + end + + def templates_with_latest_version + TEMPLATES_WITH_LATEST_VERSION + end end end end diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 16e7b8a7eca..ac1522b8a6c 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -10,21 +10,21 @@ module Gitlab APPLICATION_DEFAULT = 1 # Struct class representing a single Theme - Theme = Struct.new(:id, :name, :css_class, :css_filename) + Theme = Struct.new(:id, :name, :css_class, :css_filename, :primary_color) # All available Themes THEMES = [ - Theme.new(1, 'Indigo', 'ui-indigo', 'theme_indigo'), - Theme.new(6, 'Light Indigo', 'ui-light-indigo', 'theme_light_indigo'), - Theme.new(4, 'Blue', 'ui-blue', 'theme_blue'), - Theme.new(7, 'Light Blue', 'ui-light-blue', 'theme_light_blue'), - Theme.new(5, 'Green', 'ui-green', 'theme_green'), - Theme.new(8, 'Light Green', 'ui-light-green', 'theme_light_green'), - Theme.new(9, 'Red', 'ui-red', 'theme_red'), - Theme.new(10, 'Light Red', 'ui-light-red', 'theme_light_red'), - Theme.new(2, 'Dark', 'ui-dark', 'theme_dark'), - Theme.new(3, 'Light', 'ui-light', 'theme_light'), - Theme.new(11, 'Dark Mode (alpha)', 'gl-dark', nil) + Theme.new(1, 'Indigo', 'ui-indigo', 'theme_indigo', '#292961'), + Theme.new(6, 'Light Indigo', 'ui-light-indigo', 'theme_light_indigo', '#4b4ba3'), + Theme.new(4, 'Blue', 'ui-blue', 'theme_blue', '#1a3652'), + Theme.new(7, 'Light Blue', 'ui-light-blue', 'theme_light_blue', '#2261a1'), + Theme.new(5, 'Green', 'ui-green', 'theme_green', '#0d4524'), + Theme.new(8, 'Light Green', 'ui-light-green', 'theme_light_green', '#156b39'), + Theme.new(9, 'Red', 'ui-red', 'theme_red', '#691a16'), + Theme.new(10, 'Light Red', 'ui-light-red', 'theme_light_red', '#a62e21'), + Theme.new(2, 'Dark', 'ui-dark', 'theme_dark', '#303030'), + Theme.new(3, 'Light', 'ui-light', 'theme_light', '#666'), + Theme.new(11, 'Dark Mode (alpha)', 'gl-dark', nil, '#303030') ].freeze # Convenience method to get a space-separated String of all the theme diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb index bfdfb01093f..67ecf498cf7 100644 --- a/lib/gitlab/time_tracking_formatter.rb +++ b/lib/gitlab/time_tracking_formatter.rb @@ -24,6 +24,12 @@ module Gitlab end def output(seconds) + seconds.to_i < 0 ? negative_output(seconds) : positive_output(seconds) + end + + private + + def positive_output(seconds) ChronicDuration.output( seconds, CUSTOM_DAY_AND_MONTH_LENGTH.merge( @@ -34,7 +40,9 @@ module Gitlab nil end - private + def negative_output(seconds) + "-" + positive_output(seconds.abs) + end def limit_to_hours_setting Gitlab::CurrentSettings.time_tracking_limit_to_hours diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 4c40bfbc06f..3ec06fba5d1 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -22,9 +22,7 @@ module Gitlab }.freeze class Aggregate - delegate :weekly_time_range, - :monthly_time_range, - to: Gitlab::UsageDataCounters::HLLRedisCounter + include Gitlab::Usage::TimeFrame def initialize(recorded_at) @aggregated_metrics = load_metrics(AGGREGATED_METRICS_PATH) @@ -32,15 +30,15 @@ module Gitlab end def all_time_data - aggregated_metrics_data(start_date: nil, end_date: nil, time_frame: Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME) + aggregated_metrics_data(start_date: nil, end_date: nil, time_frame: Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME) end def monthly_data - aggregated_metrics_data(**monthly_time_range.merge(time_frame: Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME)) + aggregated_metrics_data(**monthly_time_range.merge(time_frame: Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME)) end def weekly_data - aggregated_metrics_data(**weekly_time_range.merge(time_frame: Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME)) + aggregated_metrics_data(**weekly_time_range.merge(time_frame: Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME)) end private @@ -54,7 +52,7 @@ module Gitlab case aggregation[:source] when REDIS_SOURCE - if time_frame == Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME + if time_frame == Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK Gitlab::ErrorTracking .track_and_raise_for_dev_exception( @@ -64,8 +62,6 @@ module Gitlab data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) end when DATABASE_SOURCE - next unless Feature.enabled?('database_sourced_aggregated_metrics', default_enabled: false, type: :development) - data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) else Gitlab::ErrorTracking diff --git a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb index 3069afab147..eccf79b9703 100644 --- a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb +++ b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb @@ -56,15 +56,15 @@ module Gitlab end def time_period_to_human_name(time_period) - return Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME if time_period.blank? + return Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME if time_period.blank? start_date = time_period.first.to_date end_date = time_period.last.to_date if (end_date - start_date).to_i > 7 - Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME + Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME else - Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME + Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb index 29b44f2bd0a..7b5bee3f8bd 100644 --- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb @@ -6,11 +6,14 @@ module Gitlab module Instrumentations class BaseMetric include Gitlab::Utils::UsageData + include Gitlab::Usage::TimeFrame attr_reader :time_frame + attr_reader :options - def initialize(time_frame:) + def initialize(time_frame:, options: {}) @time_frame = time_frame + @options = options end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_users_using_approve_quick_action_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_users_using_approve_quick_action_metric.rb deleted file mode 100644 index 9c92f2e9595..00000000000 --- a/lib/gitlab/usage/metrics/instrumentations/count_users_using_approve_quick_action_metric.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module Instrumentations - class CountUsersUsingApproveQuickActionMetric < RedisHLLMetric - event_names :i_quickactions_approve - end - end - end - end -end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index f83f90dea03..69a288e5b6e 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -43,16 +43,28 @@ module Gitlab finish: self.class.metric_finish&.call) end - def relation - self.class.metric_relation.call.where(time_constraints) + def to_sql + Gitlab::Usage::Metrics::Query.for(self.class.metric_operation, relation, self.class.column) + end + + def suggested_name + Gitlab::Usage::Metrics::NameSuggestion.for( + self.class.metric_operation, + relation: relation, + column: self.class.column + ) end private + def relation + self.class.metric_relation.call.where(time_constraints) + end + def time_constraints case time_frame when '28d' - { created_at: 30.days.ago..2.days.ago } + monthly_time_range_db_params when 'all' {} when 'none' diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb index 7c97cc37d17..1849773e33d 100644 --- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb @@ -13,6 +13,9 @@ module Gitlab # end # end class << self + attr_reader :metric_operation + @metric_operation = :alt + def value(&block) @metric_value = block end @@ -25,6 +28,12 @@ module Gitlab self.class.metric_value.call end end + + def suggested_name + Gitlab::Usage::Metrics::NameSuggestion.for( + self.class.metric_operation + ) + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb index 140d56f0d42..a36e612a1cb 100644 --- a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb @@ -7,35 +7,50 @@ module Gitlab class RedisHLLMetric < BaseMetric # Usage example # - # class CountUsersVisitingAnalyticsValuestreamMetric < RedisHLLMetric - # event_names :g_analytics_valuestream + # In metric YAML defintion + # instrumentation_class: RedisHLLMetric + # events: + # - g_analytics_valuestream # end class << self - def event_names(events = nil) - @metric_events = events - end + attr_reader :metric_operation + @metric_operation = :redis + end - attr_reader :metric_events + def initialize(time_frame:, options: {}) + super + + raise ArgumentError, "options events are required" unless metric_events.present? + end + + def metric_events + options[:events] end def value redis_usage_data do - event_params = time_constraints.merge(event_names: self.class.metric_events) + event_params = time_constraints.merge(event_names: metric_events) Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(**event_params) end end + def suggested_name + Gitlab::Usage::Metrics::NameSuggestion.for( + self.class.metric_operation + ) + end + private def time_constraints case time_frame when '28d' - { start_date: 4.weeks.ago.to_date, end_date: Date.current } + monthly_time_range when '7d' - { start_date: 7.days.ago.to_date, end_date: Date.current } + weekly_time_range else - raise "Unknown time frame: #{time_frame} for TimeConstraint" + raise "Unknown time frame: #{time_frame} for RedisHLLMetric" end end end diff --git a/lib/gitlab/usage/metrics/name_suggestion.rb b/lib/gitlab/usage/metrics/name_suggestion.rb new file mode 100644 index 00000000000..0728af9e2ca --- /dev/null +++ b/lib/gitlab/usage/metrics/name_suggestion.rb @@ -0,0 +1,200 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + class NameSuggestion + FREE_TEXT_METRIC_NAME = "<please fill metric name>" + REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>" + CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>" + + class << self + def for(operation, relation: nil, column: nil) + case operation + when :count + name_suggestion(column: column, relation: relation, prefix: 'count') + when :distinct_count + name_suggestion(column: column, relation: relation, prefix: 'count_distinct', distinct: :distinct) + when :estimate_batch_distinct_count + name_suggestion(column: column, relation: relation, prefix: 'estimate_distinct_count') + when :sum + name_suggestion(column: column, relation: relation, prefix: 'sum') + when :redis + REDIS_EVENT_METRIC_NAME + when :alt + FREE_TEXT_METRIC_NAME + else + raise ArgumentError, "#{operation} operation not supported" + end + end + + private + + def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil) + # rubocop: disable CodeReuse/ActiveRecord + relation = relation.unscope(where: :created_at) + # rubocop: enable CodeReuse/ActiveRecord + + parts = [prefix] + arel_column = arelize_column(relation, column) + + # nil as column indicates that the counting would use fallback value of primary key. + # Because counting primary key from relation is the conceptual equal to counting all + # records from given relation, in order to keep name suggestion more condensed + # primary key column is skipped. + # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not + # as count_id_from_issues since it does not add more information to the name suggestion + if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key] + parts << arel_column.name + parts << 'from' + end + + arel = arel_query(relation: relation, column: arel_column, distinct: distinct) + constraints = parse_constraints(relation: relation, arel: arel) + + # In some cases due to performance reasons metrics are instrumented with joined relations + # where relation listed in FROM statement is not the one that includes counted attribute + # in such situations to make name suggestion more intuitive source should be inferred based + # on the relation that provide counted attribute + # EG: SELECT COUNT(deployments.environment_id) FROM clusters + # JOIN deployments ON deployments.cluster_id = cluster.id + # should be translated into: + # count_environment_id_from_deployments_with_clusters + # instead of + # count_environment_id_from_clusters_with_deployments + actual_source = parse_source(relation, arel_column) + + append_constraints_prompt(actual_source, [constraints], parts) + + parts << actual_source + parts += process_joined_relations(actual_source, arel, relation, constraints) + parts.compact.join('_').delete('"') + end + + def append_constraints_prompt(target, constraints, parts) + applicable_constraints = constraints.select { |constraint| constraint.include?(target) } + return unless applicable_constraints.any? + + parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') } + end + + def parse_constraints(relation:, arel:) + connection = relation.connection + ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints + .new(connection) + .accept(arel, collector(connection)) + .value + end + + # TODO: joins with `USING` keyword + def process_joined_relations(actual_source, arel, relation, where_constraints) + joins = parse_joins(connection: relation.connection, arel: arel) + return [] unless joins.any? + + sources = [relation.table_name, *joins.map { |join| join[:source] }] + joins = extract_joins_targets(joins, sources) + + relations = if actual_source != relation.table_name + build_relations_tree(joins + [{ source: relation.table_name }], actual_source) + else + # in case where counter attribute comes from joined relations, the relations + # diagram has to be built bottom up, thus source and target are reverted + build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source) + end + + collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints) + end + + def parse_joins(connection:, arel:) + ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins + .new(connection) + .accept(arel) + end + + def extract_joins_targets(joins, sources) + joins.map do |join| + source_regex = /(#{join[:source]})\.(\w+_)*id/i + + tables_except_src = (sources - [join[:source]]).join('|') + target_regex = /(?<target>#{tables_except_src})\.(\w+_)*id/i + + join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i + matched = join_cond_regex.match(join[:constraints]) + + if matched + join[:target] = matched[:target] + join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '') + end + + join + end + end + + def build_relations_tree(joins, parent, source_key: :source, target_key: :target) + return [] if joins.blank? + + tree = {} + tree[parent] = [] + + joins.each do |join| + if join[source_key] == parent + tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key) + end + end + tree + end + + def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle) + conjunction = conjunctions.next + relations.each do |subtree| + subtree.each do |parent, children| + parts << "<#{conjunction}>" + join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints) + append_constraints_prompt(parent, [wheres, join_constraints].compact, parts) + parts << parent + collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions) + end + end + parts + end + + def arelize_column(relation, column) + case column + when Arel::Attribute + column + when NilClass + Arel::Table.new(relation.table_name)[relation.primary_key] + when String + if column.include?('.') + table, col = column.split('.') + Arel::Table.new(table)[col] + else + Arel::Table.new(relation.table_name)[column] + end + when Symbol + arelize_column(relation, column.to_s) + end + end + + def parse_source(relation, column) + column.relation.name || relation.table_name + end + + def collector(connection) + Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new) + end + + def arel_query(relation:, column: nil, distinct: nil) + column ||= relation.primary_key + + if column.is_a?(Arel::Attribute) + relation.select(column.count(distinct)).arel + else + relation.select(relation.all.table[column].count(distinct)).arel + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb index 49581169452..a669b43f395 100644 --- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb +++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb @@ -5,10 +5,6 @@ module Gitlab module Metrics module NamesSuggestions class Generator < ::Gitlab::UsageData - FREE_TEXT_METRIC_NAME = "<please fill metric name>" - REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>" - CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>" - class << self def generate(key_path) uncached_data.deep_stringify_keys.dig(*key_path.split('.')) @@ -17,200 +13,36 @@ module Gitlab private def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - name_suggestion(column: column, relation: relation, prefix: 'count') + Gitlab::Usage::Metrics::NameSuggestion.for(:count, column: column, relation: relation) end def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - name_suggestion(column: column, relation: relation, prefix: 'count_distinct', distinct: :distinct) + Gitlab::Usage::Metrics::NameSuggestion.for(:distinct_count, column: column, relation: relation) end def redis_usage_counter - REDIS_EVENT_METRIC_NAME + Gitlab::Usage::Metrics::NameSuggestion.for(:redis) end def alt_usage_data(*) - FREE_TEXT_METRIC_NAME + Gitlab::Usage::Metrics::NameSuggestion.for(:alt) end def redis_usage_data_totals(counter) - counter.fallback_totals.transform_values { |_| REDIS_EVENT_METRIC_NAME } + counter.fallback_totals.transform_values { |_| Gitlab::Usage::Metrics::NameSuggestion.for(:redis) } end def sum(relation, column, *rest) - name_suggestion(column: column, relation: relation, prefix: 'sum') + Gitlab::Usage::Metrics::NameSuggestion.for(:sum, column: column, relation: relation) end def estimate_batch_distinct_count(relation, column = nil, *rest) - name_suggestion(column: column, relation: relation, prefix: 'estimate_distinct_count') + Gitlab::Usage::Metrics::NameSuggestion.for(:estimate_batch_distinct_count, column: column, relation: relation) end def add(*args) "add_#{args.join('_and_')}" end - - def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil) - # rubocop: disable CodeReuse/ActiveRecord - relation = relation.unscope(where: :created_at) - # rubocop: enable CodeReuse/ActiveRecord - - parts = [prefix] - arel_column = arelize_column(relation, column) - - # nil as column indicates that the counting would use fallback value of primary key. - # Because counting primary key from relation is the conceptual equal to counting all - # records from given relation, in order to keep name suggestion more condensed - # primary key column is skipped. - # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not - # as count_id_from_issues since it does not add more information to the name suggestion - if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key] - parts << arel_column.name - parts << 'from' - end - - arel = arel_query(relation: relation, column: arel_column, distinct: distinct) - constraints = parse_constraints(relation: relation, arel: arel) - - # In some cases due to performance reasons metrics are instrumented with joined relations - # where relation listed in FROM statement is not the one that includes counted attribute - # in such situations to make name suggestion more intuitive source should be inferred based - # on the relation that provide counted attribute - # EG: SELECT COUNT(deployments.environment_id) FROM clusters - # JOIN deployments ON deployments.cluster_id = cluster.id - # should be translated into: - # count_environment_id_from_deployments_with_clusters - # instead of - # count_environment_id_from_clusters_with_deployments - actual_source = parse_source(relation, arel_column) - - append_constraints_prompt(actual_source, [constraints], parts) - - parts << actual_source - parts += process_joined_relations(actual_source, arel, relation, constraints) - parts.compact.join('_').delete('"') - end - - def append_constraints_prompt(target, constraints, parts) - applicable_constraints = constraints.select { |constraint| constraint.include?(target) } - return unless applicable_constraints.any? - - parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') } - end - - def parse_constraints(relation:, arel:) - connection = relation.connection - ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints - .new(connection) - .accept(arel, collector(connection)) - .value - end - - # TODO: joins with `USING` keyword - def process_joined_relations(actual_source, arel, relation, where_constraints) - joins = parse_joins(connection: relation.connection, arel: arel) - return [] unless joins.any? - - sources = [relation.table_name, *joins.map { |join| join[:source] }] - joins = extract_joins_targets(joins, sources) - - relations = if actual_source != relation.table_name - build_relations_tree(joins + [{ source: relation.table_name }], actual_source) - else - # in case where counter attribute comes from joined relations, the relations - # diagram has to be built bottom up, thus source and target are reverted - build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source) - end - - collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints) - end - - def parse_joins(connection:, arel:) - ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins - .new(connection) - .accept(arel) - end - - def extract_joins_targets(joins, sources) - joins.map do |join| - source_regex = /(#{join[:source]})\.(\w+_)*id/i - - tables_except_src = (sources - [join[:source]]).join('|') - target_regex = /(?<target>#{tables_except_src})\.(\w+_)*id/i - - join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i - matched = join_cond_regex.match(join[:constraints]) - - if matched - join[:target] = matched[:target] - join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '') - end - - join - end - end - - def build_relations_tree(joins, parent, source_key: :source, target_key: :target) - return [] if joins.blank? - - tree = {} - tree[parent] = [] - - joins.each do |join| - if join[source_key] == parent - tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key) - end - end - tree - end - - def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle) - conjunction = conjunctions.next - relations.each do |subtree| - subtree.each do |parent, children| - parts << "<#{conjunction}>" - join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints) - append_constraints_prompt(parent, [wheres, join_constraints].compact, parts) - parts << parent - collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions) - end - end - parts - end - - def arelize_column(relation, column) - case column - when Arel::Attribute - column - when NilClass - Arel::Table.new(relation.table_name)[relation.primary_key] - when String - if column.include?('.') - table, col = column.split('.') - Arel::Table.new(table)[col] - else - Arel::Table.new(relation.table_name)[column] - end - when Symbol - arelize_column(relation, column.to_s) - end - end - - def parse_source(relation, column) - column.relation.name || relation.table_name - end - - def collector(connection) - Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new) - end - - def arel_query(relation:, column: nil, distinct: nil) - column ||= relation.primary_key - - if column.is_a?(Arel::Attribute) - relation.select(column.count(distinct)).arel - else - relation.select(relation.all.table[column].count(distinct)).arel - end - end end end end diff --git a/lib/gitlab/usage/metrics/query.rb b/lib/gitlab/usage/metrics/query.rb new file mode 100644 index 00000000000..f6947c4c8ff --- /dev/null +++ b/lib/gitlab/usage/metrics/query.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + class Query + class << self + def for(operation, relation, column = nil, **extra) + case operation + when :count + count(relation, column) + when :distinct_count + distinct_count(relation, column) + when :sum + sum(relation, column) + when :estimate_batch_distinct_count + estimate_batch_distinct_count(relation, column) + when :histogram + histogram(relation, column, **extra) + else + raise ArgumentError, "#{operation} operation not supported" + end + end + + private + + def count(relation, column = nil) + raw_sql(relation, column) + end + + def distinct_count(relation, column = nil) + raw_sql(relation, column, true) + end + + def sum(relation, column) + relation.select(relation.all.table[column].sum).to_sql + end + + def estimate_batch_distinct_count(relation, column = nil) + raw_sql(relation, column, true) + end + + # rubocop: disable CodeReuse/ActiveRecord + def histogram(relation, column, buckets:, bucket_size: buckets.size) + count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) + cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) + + bucket_segments = bucket_size - 1 + width_bucket = Arel::Nodes::NamedFunction + .new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments]) + .as('buckets') + + query = cte + .table + .project(width_bucket, cte.table[:count]) + .group('buckets') + .order('buckets') + .with(cte.to_arel) + + query.to_sql + end + # rubocop: enable CodeReuse/ActiveRecord + + def raw_sql(relation, column, distinct = false) + column ||= relation.primary_key + relation.select(relation.all.table[column].count(distinct)).to_sql + end + end + end + end + end +end diff --git a/lib/gitlab/usage/time_frame.rb b/lib/gitlab/usage/time_frame.rb new file mode 100644 index 00000000000..966a087ee07 --- /dev/null +++ b/lib/gitlab/usage/time_frame.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module TimeFrame + ALL_TIME_TIME_FRAME_NAME = "all" + SEVEN_DAYS_TIME_FRAME_NAME = "7d" + TWENTY_EIGHT_DAYS_TIME_FRAME_NAME = "28d" + + def weekly_time_range + { start_date: 7.days.ago.to_date, end_date: Date.current } + end + + def monthly_time_range + { start_date: 4.weeks.ago.to_date, end_date: Date.current } + end + + # This time range is skewed for batch counter performance. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42972 + def monthly_time_range_db_params(column: :created_at) + { column => 30.days.ago..2.days.ago } + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index b1ba529d4a4..415a5bff261 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -33,6 +33,7 @@ module Gitlab class << self include Gitlab::Utils::UsageData include Gitlab::Utils::StrongMemoize + include Gitlab::Usage::TimeFrame def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do @@ -55,7 +56,7 @@ module Gitlab .merge(object_store_usage_data) .merge(topology_usage_data) .merge(usage_activity_by_stage) - .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, last_28_days_time_period)) + .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, monthly_time_range_db_params)) .merge(analytics_unique_visits_data) .merge(compliance_unique_visits_data) .merge(search_unique_visits_data) @@ -165,7 +166,6 @@ module Gitlab projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), - projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id), projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.terraform_reports, :project_id), projects_with_terraform_states: distinct_count(::Terraform::State, :project_id), protected_branches: count(ProtectedBranch), @@ -188,7 +188,6 @@ module Gitlab services_usage, usage_counters, user_preferences_usage, - ingress_modsecurity_usage, container_expiration_policies_usage, service_desk_counts, email_campaign_counts @@ -228,16 +227,17 @@ module Gitlab { counts_monthly: { # rubocop: disable UsageData/LargeTable: - deployments: deployment_count(Deployment.where(last_28_days_time_period)), - successful_deployments: deployment_count(Deployment.success.where(last_28_days_time_period)), - failed_deployments: deployment_count(Deployment.failed.where(last_28_days_time_period)), + deployments: deployment_count(Deployment.where(monthly_time_range_db_params)), + successful_deployments: deployment_count(Deployment.success.where(monthly_time_range_db_params)), + failed_deployments: deployment_count(Deployment.failed.where(monthly_time_range_db_params)), # rubocop: enable UsageData/LargeTable: - packages: count(::Packages::Package.where(last_28_days_time_period)), - personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)), - project_snippets: count(ProjectSnippet.where(last_28_days_time_period)), - projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(last_28_days_time_period), :project_id) + projects: count(Project.where(monthly_time_range_db_params), start: minimum_id(Project), finish: maximum_id(Project)), + packages: count(::Packages::Package.where(monthly_time_range_db_params)), + personal_snippets: count(PersonalSnippet.where(monthly_time_range_db_params)), + project_snippets: count(ProjectSnippet.where(monthly_time_range_db_params)), + projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(monthly_time_range_db_params), :project_id) }.merge( - snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp)) + snowplow_event_counts(monthly_time_range_db_params(column: :collector_tstamp)) ).tap do |data| data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end @@ -294,7 +294,6 @@ module Gitlab reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::IncomingEmail.enabled? }, signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? }, - ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity), grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }, gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? } } @@ -376,29 +375,6 @@ module Gitlab Gitlab::UsageData::Topology.new.topology_usage_data end - # rubocop: disable UsageData/DistinctCountByLargeForeignKey - def ingress_modsecurity_usage - ## - # This method measures usage of the Modsecurity Web Application Firewall across the entire - # instance's deployed environments. - # - # NOTE: this service is an approximation as it does not yet take into account if environment - # is enabled and only measures applications installed using GitLab Managed Apps (disregards - # CI-based managed apps). - # - # More details: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28331#note_318621786 - ## - - column = ::Deployment.arel_table[:environment_id] - { - ingress_modsecurity_logging: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_enabled.logging), column), - ingress_modsecurity_blocking: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_enabled.blocking), column), - ingress_modsecurity_disabled: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_disabled), column), - ingress_modsecurity_not_installed: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_not_installed), column) - } - end - # rubocop: enable UsageData/DistinctCountByLargeForeignKey - # rubocop: disable CodeReuse/ActiveRecord def container_expiration_policies_usage results = {} @@ -427,15 +403,15 @@ module Gitlab def services_usage # rubocop: disable UsageData/LargeTable: - Integration.available_services_names(include_dev: false).each_with_object({}) do |service_name, response| - service_type = Integration.service_name_to_type(service_name) - - response["projects_#{service_name}_active".to_sym] = count(Integration.active.where.not(project: nil).where(type: service_type)) - response["groups_#{service_name}_active".to_sym] = count(Integration.active.where.not(group: nil).where(type: service_type)) - response["templates_#{service_name}_active".to_sym] = count(Integration.active.where(template: true, type: service_type)) - response["instances_#{service_name}_active".to_sym] = count(Integration.active.where(instance: true, type: service_type)) - response["projects_inheriting_#{service_name}_active".to_sym] = count(Integration.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: service_type)) - response["groups_inheriting_#{service_name}_active".to_sym] = count(Integration.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: service_type)) + Integration.available_services_names(include_dev: false).each_with_object({}) do |name, response| + type = Integration.integration_name_to_type(name) + + response[:"projects_#{name}_active"] = count(Integration.active.where.not(project: nil).where(type: type)) + response[:"groups_#{name}_active"] = count(Integration.active.where.not(group: nil).where(type: type)) + response[:"templates_#{name}_active"] = count(Integration.active.where(template: true, type: type)) + response[:"instances_#{name}_active"] = count(Integration.active.where(instance: true, type: type)) + response[:"projects_inheriting_#{name}_active"] = count(Integration.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: type)) + response[:"groups_inheriting_#{name}_active"] = count(Integration.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: type)) end.merge(jira_usage, jira_import_usage) # rubocop: enable UsageData/LargeTable: end @@ -521,10 +497,6 @@ module Gitlab "#{platform}-#{ohai_data['platform_version']}" end - def last_28_days_time_period(column: :created_at) - { column => 30.days.ago..2.days.ago } - end - # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv def usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {}) { @@ -742,7 +714,7 @@ module Gitlab hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) } end results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics) } - results['analytics_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics, start_date: 4.weeks.ago.to_date, end_date: Date.current) } + results['analytics_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics, **monthly_time_range) } { analytics_unique_visits: results } end @@ -752,7 +724,7 @@ module Gitlab hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) } end results['compliance_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance) } - results['compliance_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance, start_date: 4.weeks.ago.to_date, end_date: Date.current) } + results['compliance_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance, **monthly_time_range) } { compliance_unique_visits: results } end @@ -760,11 +732,11 @@ module Gitlab def search_unique_visits_data events = ::Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('search') results = events.each_with_object({}) do |event, hash| - hash[event] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) } + hash[event] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event, **weekly_time_range) } end - results['search_unique_visits_for_any_target_weekly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: 7.days.ago.to_date, end_date: Date.current) } - results['search_unique_visits_for_any_target_monthly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: 4.weeks.ago.to_date, end_date: Date.current) } + results['search_unique_visits_for_any_target_weekly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, **weekly_time_range) } + results['search_unique_visits_for_any_target_monthly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, **monthly_time_range) } { search_unique_visits: results } end @@ -852,17 +824,16 @@ module Gitlab sent_emails = count(Users::InProductMarketingEmail.group(:track, :series)) clicked_emails = count(Users::InProductMarketingEmail.where.not(cta_clicked_at: nil).group(:track, :series)) - series_amount = Namespaces::InProductMarketingEmailsService::INTERVAL_DAYS.count - Users::InProductMarketingEmail.tracks.keys.each_with_object({}) do |track, result| # rubocop: enable UsageData/LargeTable: + series_amount = Namespaces::InProductMarketingEmailsService::TRACKS[track.to_sym][:interval_days].count 0.upto(series_amount - 1).map do |series| # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`. sent_count = sent_emails.is_a?(Hash) ? sent_emails.fetch([track, series], 0) : sent_emails clicked_count = clicked_emails.is_a?(Hash) ? clicked_emails.fetch([track, series], 0) : clicked_emails result["in_product_marketing_email_#{track}_#{series}_sent"] = sent_count - result["in_product_marketing_email_#{track}_#{series}_cta_clicked"] = clicked_count + result["in_product_marketing_email_#{track}_#{series}_cta_clicked"] = clicked_count unless track == 'experience' end end end @@ -917,7 +888,7 @@ module Gitlab end def project_imports(time_period) - { + counters = { gitlab_project: projects_imported_count('gitlab_project', time_period), gitlab: projects_imported_count('gitlab', time_period), github: projects_imported_count('github', time_period), @@ -928,6 +899,10 @@ module Gitlab manifest: projects_imported_count('manifest', time_period), gitlab_migration: count(::BulkImports::Entity.where(time_period).project_entity) # rubocop: disable CodeReuse/ActiveRecord } + + counters[:total] = add(*counters.values) + + counters end def projects_imported_count(from, time_period) diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml index dd66a40a48f..c72f487a442 100644 --- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -21,6 +21,7 @@ - i_package_golang_delete_package - i_package_golang_pull_package - i_package_golang_push_package +- i_package_helm_pull_package - i_package_maven_delete_package - i_package_maven_pull_package - i_package_maven_push_package diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 833eebd5d04..2a231f8fce0 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -38,6 +38,7 @@ module Gitlab # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current) class << self include Gitlab::Utils::UsageData + include Gitlab::Usage::TimeFrame # Track unique events # @@ -98,14 +99,6 @@ module Gitlab end end - def weekly_time_range - { start_date: 7.days.ago.to_date, end_date: Date.current } - end - - def monthly_time_range - { start_date: 4.weeks.ago.to_date, end_date: Date.current } - end - def known_event?(event_name) event_for(event_name).present? end diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index cc89fbd5caf..5023161a9dd 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -164,6 +164,11 @@ category: code_review aggregation: weekly # Diff settings events +- name: i_code_review_click_diff_view_setting + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data - name: i_code_review_click_single_file_mode_setting redis_slot: code_review category: code_review @@ -219,3 +224,11 @@ category: code_review aggregation: weekly feature_flag: diff_settings_usage_data +- name: i_code_review_user_load_conflict_ui + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_resolve_conflict + redis_slot: code_review + category: code_review + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index f2504396cc4..f2e45a52434 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -180,7 +180,6 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_i_testing_group_code_coverage_project_click_total - name: i_testing_load_performance_widget_total category: testing redis_slot: testing @@ -345,18 +344,15 @@ category: terraform redis_slot: terraform aggregation: weekly - feature_flag: usage_data_p_terraform_state_api_unique_users # Pipeline Authoring - name: o_pipeline_authoring_unique_users_committing_ciconfigfile category: pipeline_authoring redis_slot: pipeline_authoring aggregation: weekly - feature_flag: usage_data_unique_users_committing_ciconfigfile - name: o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile category: pipeline_authoring redis_slot: pipeline_authoring aggregation: weekly - feature_flag: usage_data_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile # Merge request widgets - name: users_expanding_secure_security_report redis_slot: secure diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml index adc5ba36ad7..f594c6a1b7c 100644 --- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -4,22 +4,18 @@ category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_jira_service - name: i_ecosystem_jira_service_cross_reference category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_jira_service - name: i_ecosystem_jira_service_list_issues category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_jira_service - name: i_ecosystem_jira_service_create_issue category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_jira_service - name: i_ecosystem_slack_service_issue_notification category: ecosystem redis_slot: ecosystem diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml index d1864cd569b..62b0d6dea86 100644 --- a/lib/gitlab/usage_data_counters/known_events/epic_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/epic_events.yml @@ -182,3 +182,9 @@ redis_slot: project_management aggregation: daily feature_flag: track_epics_activity + +- name: g_project_management_users_epic_issue_added_from_epic + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index d8ad2b538d6..e5031599dd0 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -47,6 +47,14 @@ category: user_packages aggregation: weekly redis_slot: package +- name: i_package_helm_deploy_token + category: deploy_token_packages + aggregation: weekly + redis_slot: package +- name: i_package_helm_user + category: user_packages + aggregation: weekly + redis_slot: package - name: i_package_maven_deploy_token category: deploy_token_packages aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index eb28a387a97..0d6f4b93aee 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -44,6 +44,8 @@ module Gitlab MR_INCLUDING_CI_CONFIG_ACTION = 'o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile' MR_MILESTONE_CHANGED_ACTION = 'i_code_review_user_milestone_changed' MR_LABELS_CHANGED_ACTION = 'i_code_review_user_labels_changed' + MR_LOAD_CONFLICT_UI_ACTION = 'i_code_review_user_load_conflict_ui' + MR_RESOLVE_CONFLICT_ACTION = 'i_code_review_user_resolve_conflict' class << self def track_mr_diffs_action(merge_request:) @@ -187,7 +189,6 @@ module Gitlab end def track_mr_including_ci_config(user:, merge_request:) - return unless Feature.enabled?(:usage_data_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile, user, default_enabled: :yaml) return unless merge_request.includes_ci_config? track_unique_action_by_user(MR_INCLUDING_CI_CONFIG_ACTION, user) @@ -201,6 +202,14 @@ module Gitlab track_unique_action_by_user(MR_LABELS_CHANGED_ACTION, user) end + def track_loading_conflict_ui_action(user:) + track_unique_action_by_user(MR_LOAD_CONFLICT_UI_ACTION, user) + end + + def track_resolve_conflict_action(user:) + track_unique_action_by_user(MR_RESOLVE_CONFLICT_ACTION, user) + end + private def track_unique_action_by_merge_request(action, merge_request) diff --git a/lib/gitlab/usage_data_metrics.rb b/lib/gitlab/usage_data_metrics.rb index e181da01229..dde5dde19e0 100644 --- a/lib/gitlab/usage_data_metrics.rb +++ b/lib/gitlab/usage_data_metrics.rb @@ -7,9 +7,12 @@ module Gitlab def uncached_data ::Gitlab::Usage::MetricDefinition.all.map do |definition| instrumentation_class = definition.attributes[:instrumentation_class] + options = definition.attributes[:options] if instrumentation_class.present? - metric_value = "Gitlab::Usage::Metrics::Instrumentations::#{instrumentation_class}".constantize.new(time_frame: definition.attributes[:time_frame]).value + metric_value = "Gitlab::Usage::Metrics::Instrumentations::#{instrumentation_class}".constantize.new( + time_frame: definition.attributes[:time_frame], + options: options).value metric_payload(definition.key_path, metric_value) else diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index 1c776501fdb..da01b68e8fc 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -6,43 +6,20 @@ module Gitlab class UsageDataQueries < UsageData class << self def count(relation, column = nil, *args, **kwargs) - raw_sql(relation, column) + Gitlab::Usage::Metrics::Query.for(:count, relation, column) end def distinct_count(relation, column = nil, *args, **kwargs) - raw_sql(relation, column, :distinct) - end - - def redis_usage_data(counter = nil, &block) - if block_given? - { redis_usage_data_block: block.to_s } - elsif counter.present? - { redis_usage_data_counter: counter } - end + Gitlab::Usage::Metrics::Query.for(:distinct_count, relation, column) end def sum(relation, column, *args, **kwargs) - relation.select(relation.all.table[column].sum).to_sql + Gitlab::Usage::Metrics::Query.for(:sum, relation, column) end # rubocop: disable CodeReuse/ActiveRecord def histogram(relation, column, buckets:, bucket_size: buckets.size) - count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) - cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) - - bucket_segments = bucket_size - 1 - width_bucket = Arel::Nodes::NamedFunction - .new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments]) - .as('buckets') - - query = cte - .table - .project(width_bucket, cte.table[:count]) - .group('buckets') - .order('buckets') - .with(cte.to_arel) - - query.to_sql + Gitlab::Usage::Metrics::Query.for(:histogram, relation, column, buckets: buckets, bucket_size: bucket_size) end # rubocop: enable CodeReuse/ActiveRecord @@ -50,11 +27,11 @@ module Gitlab # buckets query, because it can't be used to obtain estimations without # supplementary ruby code present in Gitlab::Database::PostgresHll::BatchDistinctCounter def estimate_batch_distinct_count(relation, column = nil, *args, **kwargs) - raw_sql(relation, column, :distinct) + Gitlab::Usage::Metrics::Query.for(:estimate_batch_distinct_count, relation, column) end def add(*args) - 'SELECT ' + args.map {|arg| "(#{arg})" }.join(' + ') + 'SELECT ' + args.map { |arg| "(#{arg})" }.join(' + ') end def maximum_id(model, column = nil) @@ -63,6 +40,14 @@ module Gitlab def minimum_id(model, column = nil) end + def redis_usage_data(counter = nil, &block) + if block_given? + { redis_usage_data_block: block.to_s } + elsif counter.present? + { redis_usage_data_counter: counter } + end + end + def jira_service_data { projects_jira_server_active: 0, @@ -73,13 +58,6 @@ module Gitlab def epics_deepest_relationship_level { epics_deepest_relationship_level: 0 } end - - private - - def raw_sql(relation, column, distinct = nil) - column ||= relation.primary_key - relation.select(relation.all.table[column].count(distinct)).to_sql - end end end end diff --git a/lib/gitlab/utils/measuring.rb b/lib/gitlab/utils/measuring.rb index ffd12c1b518..dc43d977a62 100644 --- a/lib/gitlab/utils/measuring.rb +++ b/lib/gitlab/utils/measuring.rb @@ -9,7 +9,7 @@ module Gitlab attr_writer :logger def logger - @logger ||= Logger.new(STDOUT) + @logger ||= Logger.new($stdout) end end @@ -67,7 +67,7 @@ module Gitlab def log_info(details) details = base_log_data.merge(details) - details = details.to_yaml if ActiveSupport::Logger.logger_outputs_to?(Measuring.logger, STDOUT) + details = details.to_yaml if ActiveSupport::Logger.logger_outputs_to?(Measuring.logger, $stdout) Measuring.logger.info(details) end end diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index b1ccdcb1df0..4ea5b5a87de 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -42,9 +42,6 @@ module Gitlab FALLBACK = -1 HISTOGRAM_FALLBACK = { '-1' => -1 }.freeze DISTRIBUTED_HLL_FALLBACK = -2 - ALL_TIME_TIME_FRAME_NAME = "all" - SEVEN_DAYS_TIME_FRAME_NAME = "7d" - TWENTY_EIGHT_DAYS_TIME_FRAME_NAME = "28d" MAX_BUCKET_SIZE = 100 def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) @@ -227,7 +224,7 @@ module Gitlab } # rubocop: disable CodeReuse/ActiveRecord - JiraService.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| + ::Integrations::Jira.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| counts = services.group_by do |service| # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 service_url = service.data_fields&.url || (service.properties && service.properties['url']) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e9905bae985..0f33c3aa68e 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -270,7 +270,7 @@ module Gitlab prefix: metadata['ArchivePrefix'], format: format, path: path.presence || "", - include_lfs_blobs: Feature.enabled?(:include_lfs_blobs_in_archive, default_enabled: true) + include_lfs_blobs: true ).to_proto ) } |