diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-05-19 07:33:21 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-05-19 07:33:21 +0000 |
| commit | 36a59d088eca61b834191dacea009677a96c052f (patch) | |
| tree | e4f33972dab5d8ef79e3944a9f403035fceea43f /lib/gitlab | |
| parent | a1761f15ec2cae7c7f7bbda39a75494add0dfd6f (diff) | |
| download | gitlab-ce-36a59d088eca61b834191dacea009677a96c052f.tar.gz | |
Add latest changes from gitlab-org/gitlab@15-0-stable-eev15.0.0-rc42
Diffstat (limited to 'lib/gitlab')
292 files changed, 8698 insertions, 2170 deletions
diff --git a/lib/gitlab/alert_management/payload.rb b/lib/gitlab/alert_management/payload.rb index 1b67b91e839..de34a0f5d47 100644 --- a/lib/gitlab/alert_management/payload.rb +++ b/lib/gitlab/alert_management/payload.rb @@ -4,8 +4,7 @@ module Gitlab module AlertManagement module Payload MONITORING_TOOLS = { - prometheus: 'Prometheus', - cilium: 'Cilium' + prometheus: 'Prometheus' }.freeze class << self @@ -48,5 +47,3 @@ module Gitlab end end end - -Gitlab::AlertManagement::Payload.prepend_mod_with('Gitlab::AlertManagement::Payload') diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index af695c5cfa4..d0d8d68362e 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -107,7 +107,7 @@ module Gitlab def use_aggregated_backend? group.present? && # for now it's only available on the group-level aggregation.enabled && - Feature.enabled?(:use_vsa_aggregated_tables, group, default_enabled: :yaml) + Feature.enabled?(:use_vsa_aggregated_tables, group) end def aggregation_attributes diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index b10330914ca..6ef5a1e2cd8 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -19,7 +19,8 @@ module Gitlab :job_id, :pipeline_id, :related_class, - :feature_category + :feature_category, + :artifact_size ].freeze private_constant :KNOWN_KEYS @@ -32,7 +33,8 @@ module Gitlab Attribute.new(:remote_ip, String), Attribute.new(:job, ::Ci::Build), Attribute.new(:related_class, String), - Attribute.new(:feature_category, String) + Attribute.new(:feature_category, String), + Attribute.new(:artifact, ::Ci::JobArtifact) ].freeze def self.known_keys @@ -74,6 +76,8 @@ module Gitlab assign_attributes(args) end + # rubocop: disable Metrics/CyclomaticComplexity + # rubocop: disable Metrics/PerceivedComplexity def to_lazy_hash {}.tap do |hash| hash[:user] = -> { username } if include_user? @@ -86,8 +90,11 @@ module Gitlab hash[:feature_category] = feature_category if set_values.include?(:feature_category) hash[:pipeline_id] = -> { job&.pipeline_id } if set_values.include?(:job) hash[:job_id] = -> { job&.id } if set_values.include?(:job) + hash[:artifact_size] = -> { artifact&.size } if set_values.include?(:artifact) end end + # rubocop: enable Metrics/CyclomaticComplexity + # rubocop: enable Metrics/PerceivedComplexity def use Labkit::Context.with_context(to_lazy_hash) { yield } diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 09775297def..41a6cbc2543 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -59,6 +59,8 @@ module Gitlab def throttled?(key, scope:, threshold: nil, users_allowlist: nil, peek: false) raise InvalidKeyError unless rate_limits[key] + ::Gitlab::Instrumentation::RateLimitingGates.track(key) + return false if scoped_user_in_allowlist?(scope, users_allowlist) threshold_value = threshold || threshold(key) diff --git a/lib/gitlab/audit/deploy_token_author.rb b/lib/gitlab/audit/deploy_token_author.rb new file mode 100644 index 00000000000..69b42034826 --- /dev/null +++ b/lib/gitlab/audit/deploy_token_author.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Audit + class DeployTokenAuthor < Gitlab::Audit::NullAuthor + def initialize(name: nil) + super(id: -2, name: name) + end + + # Events that are authored by a deploy token, should be + # shown as authored by `Deploy Token` in the UI. + def name + @name || _('Deploy Token') + end + end + end +end diff --git a/lib/gitlab/audit/null_author.rb b/lib/gitlab/audit/null_author.rb index 80e0c4ddf58..08be6ae6d9f 100644 --- a/lib/gitlab/audit/null_author.rb +++ b/lib/gitlab/audit/null_author.rb @@ -13,8 +13,8 @@ module Gitlab # # @param [Integer] id # @param [String] name - # - # @return [Gitlab::Audit::UnauthenticatedAuthor, Gitlab::Audit::DeletedAuthor, Gitlab::Audit::CiRunnerTokenAuthor] + # rubocop: disable Layout/LineLength + # @return [Gitlab::Audit::UnauthenticatedAuthor, Gitlab::Audit::DeletedAuthor, Gitlab::Audit::CiRunnerTokenAuthor, Gitlab::Audit::DeployTokenAuthor] def self.for(id, audit_event) name = audit_event[:author_name] || audit_event.details[:author_name] @@ -22,6 +22,8 @@ module Gitlab Gitlab::Audit::CiRunnerTokenAuthor.new(audit_event) elsif id == -1 Gitlab::Audit::UnauthenticatedAuthor.new(name: name) + elsif id == -2 + Gitlab::Audit::DeployTokenAuthor.new(name: name) else Gitlab::Audit::DeletedAuthor.new(id: id, name: name) end diff --git a/lib/gitlab/auth/otp/strategies/forti_authenticator.rb b/lib/gitlab/auth/otp/strategies/forti_authenticator.rb deleted file mode 100644 index c1433f05db2..00000000000 --- a/lib/gitlab/auth/otp/strategies/forti_authenticator.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Auth - module Otp - module Strategies - class FortiAuthenticator < Base - def validate(otp_code) - body = { username: user.username, - token_code: otp_code } - - response = Gitlab::HTTP.post( - auth_url, - headers: { 'Content-Type': 'application/json' }, - body: body.to_json, - basic_auth: api_credentials) - - # Successful authentication results in HTTP 200: OK - # https://docs.fortinet.com/document/fortiauthenticator/6.2.0/rest-api-solution-guide/704555/authentication-auth - response.ok? ? success : error_from_response(response) - rescue StandardError => ex - Gitlab::AppLogger.error(ex) - error(ex.message) - end - - private - - def auth_url - host = ::Gitlab.config.forti_authenticator.host - port = ::Gitlab.config.forti_authenticator.port - path = 'api/v1/auth/' - - "https://#{host}:#{port}/#{path}" - end - - def api_credentials - { username: ::Gitlab.config.forti_authenticator.username, - password: ::Gitlab.config.forti_authenticator.access_token } - end - end - end - end - end -end diff --git a/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb b/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb new file mode 100644 index 00000000000..9cf1b2247a7 --- /dev/null +++ b/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Otp + module Strategies + module FortiAuthenticator + class ManualOtp < Base + def validate(otp_code) + @otp_code = otp_code + + response = Gitlab::HTTP.post( + auth_url, + headers: { 'Content-Type': 'application/json' }, + body: body.to_json, + basic_auth: api_credentials) + + # Successful authentication results in HTTP 200: OK + # Manual OTP - https://docs.fortinet.com/document/fortiauthenticator/6.2.0/rest-api-solution-guide/704555/authentication-auth + response.ok? ? success : error_from_response(response) + rescue StandardError => ex + Gitlab::AppLogger.error(ex) + error(ex.message) + end + + private + + def auth_url + host = ::Gitlab.config.forti_authenticator.host + port = ::Gitlab.config.forti_authenticator.port + path = 'api/v1/auth/' + + "https://#{host}:#{port}/#{path}" + end + + def body + { username: user.username, + token_code: @otp_code } + end + + def api_credentials + { username: ::Gitlab.config.forti_authenticator.username, + password: ::Gitlab.config.forti_authenticator.access_token } + end + end + end + end + end + end +end diff --git a/lib/gitlab/auth/otp/strategies/forti_authenticator/push_otp.rb b/lib/gitlab/auth/otp/strategies/forti_authenticator/push_otp.rb new file mode 100644 index 00000000000..03cc648f7b0 --- /dev/null +++ b/lib/gitlab/auth/otp/strategies/forti_authenticator/push_otp.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Otp + module Strategies + module FortiAuthenticator + class PushOtp < Base + def validate + response = Gitlab::HTTP.post( + auth_url, + headers: { 'Content-Type': 'application/json' }, + body: body.to_json, + basic_auth: api_credentials) + + # Successful authentication results in HTTP 200: OK + # Push - https://docs.fortinet.com/document/fortiauthenticator/6.2.1/rest-api-solution-guide/943094/push-authentication-pushauth + response.ok? ? success : error_from_response(response) + rescue StandardError => ex + Gitlab::AppLogger.error(ex) + error(ex.message) + end + + private + + def auth_url + host = ::Gitlab.config.forti_authenticator.host + port = ::Gitlab.config.forti_authenticator.port + path = 'api/v1/pushauth/' + + "https://#{host}:#{port}/#{path}" + end + + def body + { username: user.username } + end + + def api_credentials + { username: ::Gitlab.config.forti_authenticator.username, + password: ::Gitlab.config.forti_authenticator.access_token } + end + end + end + end + end + end +end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 3f13a264b0a..815130aeee2 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -5,6 +5,10 @@ module Gitlab module Saml class Config class << self + def enabled? + ::AuthHelper.saml_providers.any? + end + def options Gitlab::Auth::OAuth::Provider.config_for('saml') end diff --git a/lib/gitlab/auth/saml/identity_linker.rb b/lib/gitlab/auth/saml/identity_linker.rb index 93195c3189f..a44a9c2fca5 100644 --- a/lib/gitlab/auth/saml/identity_linker.rb +++ b/lib/gitlab/auth/saml/identity_linker.rb @@ -32,3 +32,5 @@ module Gitlab end end end + +Gitlab::Auth::Saml::IdentityLinker.prepend_mod diff --git a/lib/gitlab/background_migration/.rubocop.yml b/lib/gitlab/background_migration/.rubocop.yml index 50112a51675..116c84c3759 100644 --- a/lib/gitlab/background_migration/.rubocop.yml +++ b/lib/gitlab/background_migration/.rubocop.yml @@ -50,3 +50,12 @@ Style/FrozenStringLiteralComment: Enabled: true Details: >- This removes the need for calling "freeze", reducing noise in the code. + +Migration/BackgroundMigrationBaseClass: + Enabled: true + Exclude: + - 'batching_strategies/**/*.rb' + - 'job_coordinator.rb' + - 'base_job.rb' + - 'batched_migration_job.rb' + - 'logger.rb' diff --git a/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb b/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb deleted file mode 100644 index f6b36571c90..00000000000 --- a/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfill expire_at for a range of Ci::JobArtifact - class BackfillArtifactExpiryDate - include Gitlab::Utils::StrongMemoize - - SWITCH_DATE = Date.new(2020, 06, 22).freeze - OLD_ARTIFACT_AGE = 15.months - BATCH_SIZE = 1_000 - OLD_ARTIFACT_EXPIRY_OFFSET = 3.months - RECENT_ARTIFACT_EXPIRY_OFFSET = 1.year - - # Ci::JobArtifact model - class Ci::JobArtifact < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'ci_job_artifacts' - - scope :without_expiry_date, -> { where(expire_at: nil) } - scope :before_switch, -> { where("date(created_at AT TIME ZONE 'UTC') < ?::date", SWITCH_DATE) } - scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } - scope :old, -> { where(self.arel_table[:created_at].lt(OLD_ARTIFACT_AGE.ago)) } - scope :recent, -> { where(self.arel_table[:created_at].gt(OLD_ARTIFACT_AGE.ago)) } - end - - def perform(start_id, end_id) - Ci::JobArtifact - .without_expiry_date.before_switch - .between(start_id, end_id) - .each_batch(of: BATCH_SIZE) do |batch| - batch.old.update_all(expire_at: old_artifact_expiry_date) - batch.recent.update_all(expire_at: recent_artifact_expiry_date) - end - end - - private - - def offset_date - strong_memoize(:offset_date) do - current_date = Time.current - target_date = Time.zone.local(current_date.year, current_date.month, 22, 0, 0, 0) - - current_date.day < 22 ? target_date : target_date.next_month - end - end - - def old_artifact_expiry_date - offset_date + OLD_ARTIFACT_EXPIRY_OFFSET - end - - def recent_artifact_expiry_date - offset_date + RECENT_ARTIFACT_EXPIRY_OFFSET - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb new file mode 100644 index 00000000000..b9151343d6a --- /dev/null +++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill draft column on open merge requests based on regex parsing of + # their titles. + # + class BackfillDraftStatusOnMergeRequestsWithCorrectedRegex # rubocop:disable Migration/BackgroundMigrationBaseClass + # Migration only version of MergeRequest table + class MergeRequest < ::ApplicationRecord + include EachBatch + + CORRECTED_REGEXP_STR = "^(\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP)" + + self.table_name = 'merge_requests' + + def self.eligible + where(state_id: 1) + .where(draft: false) + .where("title ~* ?", CORRECTED_REGEXP_STR) + end + end + + def perform(start_id, end_id) + eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id).pluck(:id) + + eligible_mrs.each_slice(10) do |slice| + MergeRequest.where(id: slice).update_all(draft: true) + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillDraftStatusOnMergeRequestsWithCorrectedRegex', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_group_features.rb b/lib/gitlab/background_migration/backfill_group_features.rb index 084c788c8cb..4c3af7be319 100644 --- a/lib/gitlab/background_migration/backfill_group_features.rb +++ b/lib/gitlab/background_migration/backfill_group_features.rb @@ -3,34 +3,19 @@ module Gitlab module BackgroundMigration # Backfill group_features for an array of groups - class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BaseJob - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, batch_size) - pause_ms = 0 if pause_ms < 0 - - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size, order_hint: :type) do |sub_batch| - batch_metrics.time_operation(:upsert_group_features) do - upsert_group_features(sub_batch, batch_size) - end - - sleep(pause_ms * 0.001) + class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BatchedMigrationJob + def perform(batch_size) + each_sub_batch( + operation_name: :upsert_group_features, + batching_arguments: { order_hint: :type }, + batching_scope: ->(relation) { relation.where(type: 'Group') } + ) do |sub_batch| + upsert_group_features(sub_batch, batch_size) end end - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - private - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: connection) - .where(source_key_column => start_id..stop_id) - .where(type: 'Group') - end - def upsert_group_features(relation, batch_size) connection.execute( <<~SQL diff --git a/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb new file mode 100644 index 00000000000..de52629522b --- /dev/null +++ b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Enable SSL verification for CI integrations with known-good hostnames. + class BackfillIntegrationsEnableSslVerification + INTEGRATIONS = { + # This matches the logic in `Integrations::DroneCi#url_is_saas?` + # - https://gitlab.com/gitlab-org/gitlab/blob/65b7fc1ad1ad33247890324e9a3396993b7718a1/app/models/integrations/drone_ci.rb#L122-127 + # - https://docs.drone.io/pipeline/environment/reference/drone-system-hostname/ + 'Integrations::DroneCi' => [ + :drone_url, + /\Acloud\.drone\.io\z/i.freeze + ], + # This matches the logic in `Integrations::Teamcity#url_is_saas?` + # - https://gitlab.com/gitlab-org/gitlab/blob/65b7fc1ad1ad33247890324e9a3396993b7718a1/app/models/integrations/teamcity.rb#L117-122 + # - https://www.jetbrains.com/help/teamcity/cloud/migrate-from-teamcity-on-premises-to-teamcity-cloud.html#Migration+Process + 'Integrations::Teamcity' => [ + :teamcity_url, + /\A[^\.]+\.teamcity\.com\z/i.freeze + ] + + # Other CI integrations which don't seem to have a SaaS offering: + # - Atlassian Bamboo (the SaaS offering is Bitbucket Pipelines) + # - Jenkins (self-hosted only) + # - MockCi (development only) + }.freeze + + # Define the `Integration` model + class Integration < ::ApplicationRecord + include IgnorableColumns + + self.table_name = :integrations + self.inheritance_column = :_type_disabled + + ignore_column :template, remove_with: '15.0', remove_after: '2022-04-22' + ignore_column :type, remove_with: '15.0', remove_after: '2022-04-22' + ignore_column :properties, remove_with: '15.1', remove_after: '2022-05-22' + + scope :affected, -> { where(type_new: INTEGRATIONS.keys).where.not(encrypted_properties: nil) } + + attr_encrypted :properties, + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm', + marshal: true, + marshaler: ::Gitlab::Json, + encode: false, + encode_iv: false + + # Handle assignment of props with symbol keys. + # To do this correctly, we need to call the method generated by attr_encrypted. + alias_method :attr_encrypted_props=, :properties= + private :attr_encrypted_props= + + def properties=(props) + self.attr_encrypted_props = props&.with_indifferent_access&.freeze + end + end + + def perform(start_id, stop_id) + integration_ids = Integration + .affected + .where(id: (start_id..stop_id)) + .pluck(:id) + + integration_ids.each do |id| + Integration.transaction do + integration = Integration.lock.find(id) + process_integration(integration) + end + end + + mark_job_as_succeeded(start_id, stop_id) + end + + private + + def process_integration(integration) + url_field, known_hostnames = INTEGRATIONS.fetch(integration.type_new) + + url = integration.properties[url_field.to_s] if integration.properties.present? + return unless url.present? + + parsed_url = Addressable::URI.parse(url) + return unless parsed_url.scheme == 'https' && parsed_url.hostname =~ known_hostnames + + integration.properties = integration.properties.merge('enable_ssl_verification' => true) + + integration.save!(touch: false) + rescue Addressable::URI::InvalidURIError, ActiveRecord::RecordInvalid + # Don't change the configuration if the record is invalid, in this case + # they will just keep having SSL verification disabled. + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_integrations_type_new.rb b/lib/gitlab/background_migration/backfill_integrations_type_new.rb index a234cebfce5..6f33472af7d 100644 --- a/lib/gitlab/background_migration/backfill_integrations_type_new.rb +++ b/lib/gitlab/background_migration/backfill_integrations_type_new.rb @@ -22,7 +22,7 @@ module Gitlab private def connection - ActiveRecord::Base.connection + ApplicationRecord.connection end def process_sub_batch(sub_batch) diff --git a/lib/gitlab/background_migration/backfill_issue_search_data.rb b/lib/gitlab/background_migration/backfill_issue_search_data.rb index ec206cbfd41..e408fd0cda6 100644 --- a/lib/gitlab/background_migration/backfill_issue_search_data.rb +++ b/lib/gitlab/background_migration/backfill_issue_search_data.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Database::DynamicModelHelpers def perform(start_id, stop_id, batch_table, batch_column, sub_batch_size, pause_ms) - define_batchable_model(batch_table, connection: ActiveRecord::Base.connection).where(batch_column => start_id..stop_id).each_batch(of: sub_batch_size) do |sub_batch| + define_batchable_model(batch_table, connection: ApplicationRecord.connection).where(batch_column => start_id..stop_id).each_batch(of: sub_batch_size) do |sub_batch| update_search_data(sub_batch) sleep(pause_ms * 0.001) diff --git a/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb b/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb index 1ed147d67c7..5f3d830c48d 100644 --- a/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb +++ b/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb @@ -26,7 +26,7 @@ module Gitlab private def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: ActiveRecord::Base.connection) + define_batchable_model(source_table, connection: ApplicationRecord.connection) .joins('INNER JOIN namespaces ON members.source_id = namespaces.id') .where(source_key_column => start_id..stop_id) .where(type: 'GroupMember') diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb index fe3edd3322b..0585924cb7b 100644 --- a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb +++ b/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb @@ -27,7 +27,7 @@ module Gitlab private def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: ActiveRecord::Base.connection) + define_batchable_model(source_table, connection: ApplicationRecord.connection) .joins('inner join namespaces on routes.source_id = namespaces.id') .where(source_key_column => start_id..stop_id) .where(namespace_id: nil) diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb index f6c8fb060f8..0282531ae17 100644 --- a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb +++ b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb @@ -13,7 +13,7 @@ module Gitlab cleanup_gin_index('routes') batch_metrics.time_operation(:update_all) do - ActiveRecord::Base.connection.execute <<~SQL + ApplicationRecord.connection.execute <<~SQL WITH route_and_ns(route_id, project_namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{sub_batch.to_sql} ) @@ -48,7 +48,7 @@ module Gitlab end def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: ActiveRecord::Base.connection) + define_batchable_model(source_table, connection: ApplicationRecord.connection) .joins('INNER JOIN projects ON routes.source_id = projects.id') .where(source_key_column => start_id..stop_id) .where(namespace_id: nil) diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb index 79e7a2f2279..587de1bcb5a 100644 --- a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb +++ b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb @@ -29,7 +29,7 @@ module Gitlab WHERE namespaces.id = calculated_ids.id AND namespaces.traversal_ids = '{}' SQL - ActiveRecord::Base.connection.execute(update_sql) + ApplicationRecord.connection.execute(update_sql) sleep PAUSE_SECONDS end diff --git a/lib/gitlab/background_migration/backfill_note_discussion_id.rb b/lib/gitlab/background_migration/backfill_note_discussion_id.rb new file mode 100644 index 00000000000..da2c31ebd11 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_note_discussion_id.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Fixes notes with NULL discussion_ids due to a bug when importing from GitHub + # Bug was fixed in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/76517 + class BackfillNoteDiscussionId + SUB_BATCH_SIZE = 300 + + # Migration only version of notes model + class Note < ApplicationRecord + include EachBatch + + self.table_name = 'notes' + + # Based on https://gitlab.com/gitlab-org/gitlab/blob/117c14d0c79403e169cf52922b48f69d1dcf6a85/app/models/discussion.rb#L62-74 + def generate_discussion_id + Digest::SHA1.hexdigest( + [:discussion, noteable_type.try(:underscore), noteable_id || commit_id, SecureRandom.hex].join('-') + ) + end + end + + def perform(start_id, stop_id) + notes = Note.select(:id, :noteable_type, :noteable_id, :commit_id) + .where(discussion_id: nil, id: start_id..stop_id) + + notes.each_batch(of: SUB_BATCH_SIZE) do |relation| + update_discussion_ids(relation) + end + end + + private + + def update_discussion_ids(notes) + mapping = notes.each_with_object({}) do |note, hash| + hash[note] = { discussion_id: note.generate_discussion_id } + end + + Gitlab::Database::BulkUpdate.execute(%i(discussion_id), mapping) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_settings.rb b/lib/gitlab/background_migration/backfill_project_settings.rb new file mode 100644 index 00000000000..7ede8de7bd6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_settings.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Back-fill project settings for projects that do not yet have one. + class BackfillProjectSettings + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) + batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + + batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + insert_sql = <<~SQL + INSERT INTO project_settings (project_id, created_at, updated_at) + #{sub_batch.where(project_settings: { project_id: nil }) + .select('projects.id, NOW(), NOW()') + .to_sql} + ON CONFLICT (project_id) DO NOTHING + SQL + + connection.execute(insert_sql) + + pause_ms = 0 if pause_ms < 0 + sleep(pause_ms * 0.001) + end + end + + private + + def connection + ApplicationRecord.connection + end + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(:projects, connection: connection) + .where(source_key_column => start_id..stop_id) + .joins("LEFT OUTER JOIN project_settings ON project_settings.project_id = projects.id") + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_topics_title.rb b/lib/gitlab/background_migration/backfill_topics_title.rb new file mode 100644 index 00000000000..19a1eff5b58 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_topics_title.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to backfill the topic title + class BackfillTopicsTitle + # Temporary AR model for topics + class Topic < ActiveRecord::Base + self.table_name = 'topics' + end + + def perform(start_id, end_id) + Topic.where(id: start_id..end_id).where(title: nil).update_all('title = name') + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb b/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb index 170af90805a..3bf6bf993dd 100644 --- a/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb +++ b/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb @@ -16,7 +16,7 @@ module Gitlab private def execute(sql) - @connection ||= ::ActiveRecord::Base.connection + @connection ||= ApplicationRecord.connection @connection.execute(sql) end diff --git a/lib/gitlab/background_migration/backfill_user_namespace.rb b/lib/gitlab/background_migration/backfill_user_namespace.rb index ab569e236fb..df6b1f083c3 100644 --- a/lib/gitlab/background_migration/backfill_user_namespace.rb +++ b/lib/gitlab/background_migration/backfill_user_namespace.rb @@ -25,7 +25,7 @@ module Gitlab private def connection - ActiveRecord::Base.connection + ApplicationRecord.connection end def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb new file mode 100644 index 00000000000..442eab0673e --- /dev/null +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Base class for batched background migrations. Subclasses should implement the `#perform` + # method as the entry point for the job's execution, which will be called with the migration + # arguments (if any). + class BatchedMigrationJob + include Gitlab::Database::DynamicModelHelpers + + def initialize(start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, connection:) + @start_id = start_id + @end_id = end_id + @batch_table = batch_table + @batch_column = batch_column + @sub_batch_size = sub_batch_size + @pause_ms = pause_ms + @connection = connection + end + + def perform(*job_arguments) + raise NotImplementedError, "subclasses of #{self.class.name} must implement #{__method__}" + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + attr_reader :start_id, :end_id, :batch_table, :batch_column, :sub_batch_size, :pause_ms, :connection + + def each_sub_batch(operation_name: :default, batching_arguments: {}, batching_scope: nil) + all_batching_arguments = { column: batch_column, of: sub_batch_size }.merge(batching_arguments) + + parent_relation = parent_batch_relation(batching_scope) + + parent_relation.each_batch(**all_batching_arguments) do |relation| + batch_metrics.instrument_operation(operation_name) do + yield relation + end + + sleep([pause_ms, 0].max * 0.001) + end + end + + def parent_batch_relation(batching_scope) + parent_relation = define_batchable_model(batch_table, connection: connection) + .where(batch_column => start_id..end_id) + + return parent_relation unless batching_scope + + batching_scope.call(parent_relation) + end + end + end +end diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index 137b4d4bc4e..826845935b8 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -13,50 +13,25 @@ module Gitlab # - We skip the NULL checks as they may result in not using an index scan # - The table that is migrated does _not_ need `id` as the primary key # We use the provided primary_key column to perform the update. - class CopyColumnUsingBackgroundMigrationJob < BaseJob - include Gitlab::Database::DynamicModelHelpers + class CopyColumnUsingBackgroundMigrationJob < BatchedMigrationJob + def perform(copy_from, copy_to) + assignment_clauses = build_assignment_clauses(copy_from, copy_to) - # start_id - The start ID of the range of rows to update. - # end_id - The end ID of the range of rows to update. - # batch_table - The name of the table that contains the columns. - # batch_column - The name of the column we use to batch over the table. - # sub_batch_size - We don't want updates to take more than ~100ms - # This allows us to run multiple smaller batches during - # the minimum 2.minute interval that we can schedule jobs - # pause_ms - The number of milliseconds to sleep between each subbatch execution. - # copy_from - List of columns containing the data to copy. - # copy_to - List of columns to copy the data to. Order must match the order in `copy_from`. - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, copy_from, copy_to) - copy_from = Array.wrap(copy_from) - copy_to = Array.wrap(copy_to) - - raise ArgumentError, 'number of source and destination columns must match' unless copy_from.count == copy_to.count - - assignment_clauses = column_assignment_clauses(copy_from, copy_to) - - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:update_all) do - sub_batch.update_all(assignment_clauses) - end - - pause_ms = 0 if pause_ms < 0 - sleep(pause_ms * 0.001) + each_sub_batch(operation_name: :update_all) do |relation| + relation.update_all(assignment_clauses) end end - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - private - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: connection).where(source_key_column => start_id..stop_id) - end + def build_assignment_clauses(copy_from, copy_to) + copy_from = Array.wrap(copy_from) + copy_to = Array.wrap(copy_to) + + unless copy_from.count == copy_to.count + raise ArgumentError, 'number of source and destination columns must match' + end - def column_assignment_clauses(copy_from, copy_to) assignments = copy_from.zip(copy_to).map do |from_column, to_column| from_column = connection.quote_column_name(from_column) to_column = connection.quote_column_name(to_column) diff --git a/lib/gitlab/background_migration/delete_orphaned_deployments.rb b/lib/gitlab/background_migration/delete_orphaned_deployments.rb index 5d41a46c8cd..4a3a12ab53d 100644 --- a/lib/gitlab/background_migration/delete_orphaned_deployments.rb +++ b/lib/gitlab/background_migration/delete_orphaned_deployments.rb @@ -15,7 +15,7 @@ module Gitlab end def orphaned_deployments - define_batchable_model('deployments', connection: ActiveRecord::Base.connection) + define_batchable_model('deployments', connection: ApplicationRecord.connection) .where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') end diff --git a/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb b/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb index 9a88eb8ea06..dad5da875ab 100644 --- a/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb +++ b/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb @@ -32,7 +32,7 @@ module Gitlab private def execute(sql) - ActiveRecord::Base + ApplicationRecord .connection .execute(sql) end diff --git a/lib/gitlab/background_migration/expire_o_auth_tokens.rb b/lib/gitlab/background_migration/expire_o_auth_tokens.rb new file mode 100644 index 00000000000..595e4ac9dc8 --- /dev/null +++ b/lib/gitlab/background_migration/expire_o_auth_tokens.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Add expiry to all OAuth access tokens + class ExpireOAuthTokens < ::Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + each_sub_batch( + operation_name: :update_oauth_tokens, + batching_scope: ->(relation) { relation.where(expires_in: nil) } + ) do |sub_batch| + update_oauth_tokens(sub_batch) + end + end + + private + + def update_oauth_tokens(relation) + relation.update_all(expires_in: 7_200) + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb b/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb index defd9ea832b..3772430d0b7 100644 --- a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb +++ b/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb @@ -21,7 +21,7 @@ module Gitlab backfill_project_namespaces_service.cleanup_gin_index('projects') project_ids.each_slice(SUB_BATCH_SIZE) do |ids| - ActiveRecord::Base.connection.execute(update_projects_name_and_path_sql(ids)) + ApplicationRecord.connection.execute(update_projects_name_and_path_sql(ids)) end backfill_project_namespaces_service.backfill_project_namespaces diff --git a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb index 83c01afa432..c21f9c1d50f 100644 --- a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb +++ b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb @@ -14,7 +14,7 @@ module Gitlab private def create_missing!(from_id, to_id) - result = ActiveRecord::Base.connection.select_one(sql(from_id, to_id)) + result = ApplicationRecord.connection.select_one(sql(from_id, to_id)) return 0 unless result result['number_of_created_records'] diff --git a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb index b8e4562b3bf..496ec0bd0a1 100644 --- a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb +++ b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb @@ -120,14 +120,14 @@ module Gitlab end def create_missing(from_id, to_id) - result = ActiveRecord::Base.connection.select_one(create_sql(from_id, to_id)) + result = ApplicationRecord.connection.select_one(create_sql(from_id, to_id)) return unless result logger.info(message: "#{self.class}: created missing services for #{result['number_of_created_records']} projects in id=#{from_id}...#{to_id}") end def update_inconsistent(from_id, to_id) - result = ActiveRecord::Base.connection.select_one(update_sql(from_id, to_id)) + result = ApplicationRecord.connection.select_one(update_sql(from_id, to_id)) return unless result logger.info(message: "#{self.class}: updated inconsistent services for #{result['number_of_updated_records']} projects in id=#{from_id}...#{to_id}") diff --git a/lib/gitlab/background_migration/job_coordinator.rb b/lib/gitlab/background_migration/job_coordinator.rb index acbb5f76ad8..c440db58b94 100644 --- a/lib/gitlab/background_migration/job_coordinator.rb +++ b/lib/gitlab/background_migration/job_coordinator.rb @@ -14,7 +14,7 @@ module Gitlab worker_class = worker_for_tracking_database[tracking_database] if worker_class.nil? - raise ArgumentError, "tracking_database must be one of [#{worker_for_tracking_database.keys.join(', ')}]" + raise ArgumentError, "The '#{tracking_database}' must be one of #{worker_for_tracking_database.keys.to_a}" end new(worker_class) diff --git a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb b/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb index ec4631d1e34..d7d24960a41 100644 --- a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb +++ b/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb @@ -7,7 +7,7 @@ module Gitlab include Gitlab::Database::DynamicModelHelpers def perform(start_id, end_id) - define_batchable_model('integrations', connection: ::ActiveRecord::Base.connection) + define_batchable_model('integrations', connection: ApplicationRecord.connection) .where(id: start_id..end_id, type_new: %w[Integrations::Confluence Integrations::Shimo]) .update_all(category: 'third_party_wiki') diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb deleted file mode 100644 index 6a29a632577..00000000000 --- a/lib/gitlab/background_migration/migrate_stage_status.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Metrics/AbcSize -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MigrateStageStatus - STATUSES = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze - - class Build < ActiveRecord::Base - self.table_name = 'ci_builds' - - scope :latest, -> { where(retried: [false, nil]) } - scope :created, -> { where(status: 'created') } - scope :running, -> { where(status: 'running') } - scope :pending, -> { where(status: 'pending') } - scope :success, -> { where(status: 'success') } - scope :failed, -> { where(status: 'failed') } - scope :canceled, -> { where(status: 'canceled') } - scope :skipped, -> { where(status: 'skipped') } - scope :manual, -> { where(status: 'manual') } - - scope :failed_but_allowed, -> do - where(allow_failure: true, status: [:failed, :canceled]) - end - - scope :exclude_ignored, -> do - where("allow_failure = ? OR status IN (?)", - false, %w[created pending running success skipped]) - end - - def self.status_sql - scope_relevant = latest.exclude_ignored - scope_warnings = latest.failed_but_allowed - - builds = scope_relevant.select('count(*)').to_sql - created = scope_relevant.created.select('count(*)').to_sql - success = scope_relevant.success.select('count(*)').to_sql - manual = scope_relevant.manual.select('count(*)').to_sql - pending = scope_relevant.pending.select('count(*)').to_sql - running = scope_relevant.running.select('count(*)').to_sql - skipped = scope_relevant.skipped.select('count(*)').to_sql - canceled = scope_relevant.canceled.select('count(*)').to_sql - warnings = scope_warnings.select('count(*) > 0').to_sql - - <<-SQL.strip_heredoc - (CASE - WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]} - WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]} - WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]} - WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]} - WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]} - WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]} - WHEN (#{created}) > 0 THEN #{STATUSES[:running]} - ELSE #{STATUSES[:failed]} - END) - SQL - end - end - - def perform(start_id, stop_id) - status_sql = Build - .where('ci_builds.commit_id = ci_stages.pipeline_id') - .where('ci_builds.stage = ci_stages.name') - .status_sql - - sql = <<-SQL - UPDATE ci_stages SET status = (#{status_sql}) - WHERE ci_stages.status IS NULL - AND ci_stages.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} - SQL - - ActiveRecord::Base.connection.execute(sql) - end - end - end -end diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb index c01545e5dca..06422ed282f 100644 --- a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb +++ b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb @@ -22,7 +22,7 @@ module Gitlab private def process_batch(from_id, to_id) - ActiveRecord::Base.connection.execute(update_sql(from_id, to_id)) + ApplicationRecord.connection.execute(update_sql(from_id, to_id)) logger.info(message: "#{self.class}: Copied container_registry_enabled values for projects with IDs between #{from_id}..#{to_id}") end diff --git a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb index 78e897d9ae1..36d4e649271 100644 --- a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb +++ b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb @@ -26,7 +26,7 @@ module Gitlab private def connection - ActiveRecord::Base.connection + ::Ci::ApplicationRecord.connection end def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) diff --git a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb b/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb index 9e102ea1517..a9611e9814c 100644 --- a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb +++ b/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb @@ -33,7 +33,7 @@ module Gitlab private def connection - @connection ||= ::ActiveRecord::Base.connection + @connection ||= ApplicationRecord.connection end def execute(sql) diff --git a/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb b/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb index 769ca4be7f3..1f2b55004e4 100644 --- a/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb +++ b/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb @@ -15,7 +15,7 @@ module Gitlab def perform(start_id, stop_id) Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch| - ActiveRecord::Base.connection.execute(<<~SQL) + ApplicationRecord.connection.execute(<<~SQL) WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql}) UPDATE topics SET non_private_projects_count = ( diff --git a/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb b/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb index 1d96872d445..2495cb51364 100644 --- a/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb +++ b/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb @@ -15,7 +15,7 @@ module Gitlab def perform(start_id, stop_id) Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch| - ActiveRecord::Base.connection.execute(<<~SQL) + ApplicationRecord.connection.execute(<<~SQL) WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql}) UPDATE topics SET total_projects_count = (SELECT COUNT(*) FROM project_topics WHERE topic_id = batched_relation.id) diff --git a/lib/gitlab/background_migration/populate_vulnerability_reads.rb b/lib/gitlab/background_migration/populate_vulnerability_reads.rb index 7b6d4c1ff81..5e6475a3d1a 100644 --- a/lib/gitlab/background_migration/populate_vulnerability_reads.rb +++ b/lib/gitlab/background_migration/populate_vulnerability_reads.rb @@ -26,7 +26,7 @@ module Gitlab end def connection - ActiveRecord::Base.connection + ApplicationRecord.connection end def insert_query(start_id, end_id) diff --git a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb index c13dbd76630..2b27bad3497 100644 --- a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb +++ b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb @@ -58,7 +58,7 @@ module Gitlab index_names = ApplicationRecord.connection.select_values("select indexname::text from pg_indexes where tablename = '#{table_name}' and indexdef ilike '%using gin%'") index_names.each do |index_name| - ActiveRecord::Base.connection.execute("select gin_clean_pending_list('#{index_name}')") + ApplicationRecord.connection.execute("select gin_clean_pending_list('#{index_name}')") end end @@ -77,7 +77,7 @@ module Gitlab projects = IsolatedModels::Project.where(id: project_ids) .select("projects.id, projects.name, projects.path, projects.namespace_id, projects.visibility_level, shared_runners_enabled, '#{PROJECT_NAMESPACE_STI_NAME}', now(), now()") - ActiveRecord::Base.connection.execute <<~SQL + ApplicationRecord.connection.execute <<~SQL INSERT INTO namespaces (tmp_project_id, name, path, parent_id, visibility_level, shared_runners_enabled, type, created_at, updated_at) #{projects.to_sql} ON CONFLICT DO NOTHING; @@ -89,7 +89,7 @@ module Gitlab .joins("INNER JOIN namespaces ON projects.id = namespaces.tmp_project_id") .select("namespaces.id, namespaces.tmp_project_id") - ActiveRecord::Base.connection.execute <<~SQL + ApplicationRecord.connection.execute <<~SQL WITH cte(project_namespace_id, project_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{projects.to_sql} ) @@ -105,7 +105,7 @@ module Gitlab .joins("INNER JOIN namespaces n2 ON namespaces.parent_id = n2.id") .select("namespaces.id as project_namespace_id, n2.traversal_ids") - ActiveRecord::Base.connection.execute <<~SQL + ApplicationRecord.connection.execute <<~SQL UPDATE namespaces SET traversal_ids = array_append(project_namespaces.traversal_ids, project_namespaces.project_namespace_id) FROM (#{namespaces.to_sql}) as project_namespaces(project_namespace_id, traversal_ids) diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index c1b8de1f6aa..db7afd59f4d 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -79,7 +79,7 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # r # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength def perform(start_id, end_id) - unless Feature.enabled?(:migrate_vulnerability_finding_uuids, default_enabled: true) + unless Feature.enabled?(:migrate_vulnerability_finding_uuids) return log_info('Migration is disabled by the feature flag', start_id: start_id, end_id: end_id) end diff --git a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb index 323f109449b..4acef9029f9 100644 --- a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb +++ b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb @@ -10,7 +10,7 @@ module Gitlab include Gitlab::Database::DynamicModelHelpers def perform(start_id, stop_id) - define_batchable_model('vulnerability_finding_links', connection: ActiveRecord::Base.connection) + define_batchable_model('vulnerability_finding_links', connection: ApplicationRecord.connection) .where(id: start_id..stop_id) .delete_all end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb index 80ca76ef37f..190e2fc22fb 100644 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb @@ -5,24 +5,24 @@ module Gitlab # A job to nullify duplicate runners_token_encrypted values in projects table in batches class ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects class Project < ActiveRecord::Base # rubocop:disable Style/Documentation - include ::EachBatch + include EachBatch self.table_name = 'projects' - scope :base_query, -> do - where.not(runners_token_encrypted: nil) - end + scope :base_query, -> { where.not(runners_token_encrypted: nil) } end def perform(start_id, end_id) # Reset duplicate runner tokens that would prevent creating an unique index. + batch_records = Project.base_query.where(id: start_id..end_id) + duplicate_tokens = Project.base_query - .where(id: start_id..end_id) + .where(runners_token_encrypted: batch_records.select(:runners_token_encrypted).distinct) .group(:runners_token_encrypted) .having('COUNT(*) > 1') .pluck(:runners_token_encrypted) - Project.where(runners_token_encrypted: duplicate_tokens).update_all(runners_token_encrypted: nil) if duplicate_tokens.any? + batch_records.where(runners_token_encrypted: duplicate_tokens).update_all(runners_token_encrypted: nil) if duplicate_tokens.any? mark_job_as_succeeded(start_id, end_id) end @@ -30,7 +30,10 @@ module Gitlab private def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects', arguments) + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) end end end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb index d87ce6c88d3..b58eefa0ab3 100644 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb @@ -5,24 +5,24 @@ module Gitlab # A job to nullify duplicate ci_runners_token values in projects table in batches class ResetDuplicateCiRunnersTokenValuesOnProjects class Project < ActiveRecord::Base # rubocop:disable Style/Documentation - include ::EachBatch + include EachBatch self.table_name = 'projects' - scope :base_query, -> do - where.not(runners_token: nil) - end + scope :base_query, -> { where.not(runners_token: nil) } end def perform(start_id, end_id) # Reset duplicate runner tokens that would prevent creating an unique index. + batch_records = Project.base_query.where(id: start_id..end_id) + duplicate_tokens = Project.base_query - .where(id: start_id..end_id) + .where(runners_token: batch_records.select(:runners_token).distinct) .group(:runners_token) .having('COUNT(*) > 1') .pluck(:runners_token) - Project.where(runners_token: duplicate_tokens).update_all(runners_token: nil) if duplicate_tokens.any? + batch_records.where(runners_token: duplicate_tokens).update_all(runners_token: nil) if duplicate_tokens.any? mark_job_as_succeeded(start_id, end_id) end @@ -30,7 +30,10 @@ module Gitlab private def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('ResetDuplicateCiRunnerValuesTokensOnProjects', arguments) + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) end end end diff --git a/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports.rb b/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports.rb new file mode 100644 index 00000000000..83a7eb0b4cc --- /dev/null +++ b/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to reset container_repositories that were skipped in the phase 2 registry + # migration due to too many tags. + class ResetTooManyTagsSkippedRegistryImports # rubocop:disable Migration/BackgroundMigrationBaseClass + class ContainerRepository < ::ApplicationRecord # rubocop:disable Style/Documentation + include EachBatch + + self.table_name = 'container_repositories' + + scope :base_query, -> { where(migration_state: 'import_skipped', migration_skipped_reason: 2) } + end + + def perform(start_id, end_id) + ContainerRepository.base_query.where(id: start_id..end_id).each_batch(of: 100) do |sub_batch| + sub_batch.update_all( + migration_pre_import_started_at: nil, + migration_pre_import_done_at: nil, + migration_import_started_at: nil, + migration_import_done_at: nil, + migration_aborted_at: nil, + migration_skipped_at: nil, + migration_retries_count: 0, + migration_skipped_reason: nil, + migration_state: 'default', + migration_aborted_in_state: nil + ) + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb index f54bb8256d0..38932e52bb0 100644 --- a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb +++ b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb @@ -28,7 +28,7 @@ module Gitlab end def connection - @connection ||= ::ActiveRecord::Base.connection + @connection ||= ApplicationRecord.connection end def execute(sql) diff --git a/lib/gitlab/background_migration/update_timelogs_project_id.rb b/lib/gitlab/background_migration/update_timelogs_project_id.rb index 24c9967b88e..69bb5cf6e6d 100644 --- a/lib/gitlab/background_migration/update_timelogs_project_id.rb +++ b/lib/gitlab/background_migration/update_timelogs_project_id.rb @@ -36,7 +36,7 @@ module Gitlab end def execute(sql) - @connection ||= ::ActiveRecord::Base.connection + @connection ||= ApplicationRecord.connection @connection.execute(sql) end end diff --git a/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb b/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb index f5ba9e63333..10db9f5064a 100644 --- a/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb +++ b/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb @@ -5,7 +5,7 @@ module Gitlab module BackgroundMigration class UpdateUsersWhereTwoFactorAuthRequiredFromGroup # rubocop:disable Metrics/ClassLength def perform(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL + ApplicationRecord.connection.execute <<~SQL UPDATE users SET diff --git a/lib/gitlab/backtrace_cleaner.rb b/lib/gitlab/backtrace_cleaner.rb index caea05c720d..d2ca2057eb6 100644 --- a/lib/gitlab/backtrace_cleaner.rb +++ b/lib/gitlab/backtrace_cleaner.rb @@ -17,7 +17,6 @@ module Gitlab lib/gitlab/profiler.rb lib/gitlab/query_limiting/ lib/gitlab/request_context.rb - lib/gitlab/request_profiler/ lib/gitlab/sidekiq_logging/ lib/gitlab/sidekiq_middleware/ lib/gitlab/sidekiq_status/ diff --git a/lib/gitlab/chat.rb b/lib/gitlab/chat.rb index 23d4fb36b66..30e9989d270 100644 --- a/lib/gitlab/chat.rb +++ b/lib/gitlab/chat.rb @@ -4,7 +4,7 @@ module Gitlab module Chat # Returns `true` if Chatops is available for the current instance. def self.available? - ::Feature.enabled?(:chatops, default_enabled: true) + ::Feature.enabled?(:chatops) end end end diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb index 84c01cf4baf..2e469aabeb2 100644 --- a/lib/gitlab/checks/changes_access.rb +++ b/lib/gitlab/checks/changes_access.rb @@ -3,6 +3,8 @@ module Gitlab module Checks class ChangesAccess + include Gitlab::Utils::StrongMemoize + ATTRIBUTES = %i[user_access project protocol changes logger].freeze attr_reader(*ATTRIBUTES) @@ -33,29 +35,37 @@ module Gitlab # changes. This set may also contain commits which are not referenced by # any of the new revisions. def commits - allow_quarantine = true + strong_memoize(:commits) do + allow_quarantine = true + + newrevs = @changes.map do |change| + oldrev = change[:oldrev] + newrev = change[:newrev] - newrevs = @changes.map do |change| - oldrev = change[:oldrev] - newrev = change[:newrev] + next if blank_rev?(newrev) - next if blank_rev?(newrev) + # In case any of the old revisions is blank, then we cannot reliably + # detect which commits are new for a given change when enumerating + # objects via the object quarantine directory given that the client + # may have pushed too many commits, and we don't know when to + # terminate the walk. We thus fall back to using `git rev-list --not + # --all`, which is a lot less efficient but at least can only ever + # returns commits which really are new. + allow_quarantine = false if allow_quarantine && blank_rev?(oldrev) - # In case any of the old revisions is blank, then we cannot reliably - # detect which commits are new for a given change when enumerating - # objects via the object quarantine directory given that the client - # may have pushed too many commits, and we don't know when to - # terminate the walk. We thus fall back to using `git rev-list --not - # --all`, which is a lot less efficient but at least can only ever - # returns commits which really are new. - allow_quarantine = false if allow_quarantine && blank_rev?(oldrev) + newrev + end.compact - newrev - end.compact + next [] if newrevs.empty? - return [] if newrevs.empty? + # When filtering quarantined commits we can enable usage of the object + # quarantine no matter whether we have an `oldrev` or not. + if Feature.enabled?(:filter_quarantined_commits) + allow_quarantine = true + end - @commits ||= project.repository.new_commits(newrevs, allow_quarantine: allow_quarantine) + project.repository.new_commits(newrevs, allow_quarantine: allow_quarantine) + end end # All commits which have been newly introduced via the given revision. diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index 84069a1249b..1d1d24c8fcc 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -9,7 +9,7 @@ module Gitlab def validate! # This feature flag is used for disabling integrity check on some envs # because these costy calculations may cause performance issues - return unless Feature.enabled?(:lfs_check, project, default_enabled: :yaml) + return unless Feature.enabled?(:lfs_check, project) return unless project.lfs_enabled? diff --git a/lib/gitlab/checks/single_change_access.rb b/lib/gitlab/checks/single_change_access.rb index 2fd48dfbfe2..8e12801daee 100644 --- a/lib/gitlab/checks/single_change_access.rb +++ b/lib/gitlab/checks/single_change_access.rb @@ -35,7 +35,8 @@ module Gitlab end def commits - @commits ||= project.repository.new_commits(newrev) + @commits ||= project.repository.new_commits(newrev, + allow_quarantine: Feature.enabled?(:filter_quarantined_commits)) end protected diff --git a/lib/gitlab/ci/badge/coverage/template.rb b/lib/gitlab/ci/badge/coverage/template.rb index f12b4f2dbfb..18db4861dc9 100644 --- a/lib/gitlab/ci/badge/coverage/template.rb +++ b/lib/gitlab/ci/badge/coverage/template.rb @@ -23,13 +23,11 @@ module Gitlab::Ci MIN_MEDIUM_DEFAULT = 75 def initialize(badge) - @entity = badge.entity @status = badge.status - @key_text = badge.customization.dig(:key_text) - @key_width = badge.customization.dig(:key_width) @min_good = badge.customization.dig(:min_good) @min_acceptable = badge.customization.dig(:min_acceptable) @min_medium = badge.customization.dig(:min_medium) + super end def value_text diff --git a/lib/gitlab/ci/badge/pipeline/template.rb b/lib/gitlab/ci/badge/pipeline/template.rb index c39f96e4a34..417fff252a3 100644 --- a/lib/gitlab/ci/badge/pipeline/template.rb +++ b/lib/gitlab/ci/badge/pipeline/template.rb @@ -22,10 +22,8 @@ module Gitlab::Ci }.freeze def initialize(badge) - @entity = badge.entity @status = badge.status - @key_text = badge.customization.dig(:key_text) - @key_width = badge.customization.dig(:key_width) + super end def value_text diff --git a/lib/gitlab/ci/badge/release/template.rb b/lib/gitlab/ci/badge/release/template.rb index 65bff4371cf..354be6276fa 100644 --- a/lib/gitlab/ci/badge/release/template.rb +++ b/lib/gitlab/ci/badge/release/template.rb @@ -13,10 +13,8 @@ module Gitlab::Ci VALUE_WIDTH_DEFAULT = 54 def initialize(badge) - @entity = badge.entity @tag = badge.tag || "none" - @key_width = badge.customization.dig(:key_width) - @key_text = badge.customization.dig(:key_text) + super end def key_text diff --git a/lib/gitlab/ci/badge/template.rb b/lib/gitlab/ci/badge/template.rb index d514a8577bd..b185fadc3a2 100644 --- a/lib/gitlab/ci/badge/template.rb +++ b/lib/gitlab/ci/badge/template.rb @@ -12,7 +12,8 @@ module Gitlab::Ci def initialize(badge) @entity = badge.entity - @status = badge.status + @key_text = badge.customization.dig(:key_text) + @key_width = badge.customization.dig(:key_width) end def key_text diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 2c9524c89ff..15a4ff91c1b 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -26,11 +26,8 @@ module Gitlab @source_ref_path = pipeline&.source_ref_path @project = project - if use_config_variables? - pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) - end - @context = self.logger.instrument(:config_build_context) do + pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) end @@ -94,7 +91,8 @@ module Gitlab def metadata { - includes: @context.includes + includes: @context.includes, + merged_yaml: @config&.deep_stringify_keys&.to_yaml } end @@ -148,46 +146,15 @@ module Gitlab sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, - variables: build_variables(project: project, pipeline: pipeline), + variables: build_variables(pipeline: pipeline), logger: logger) end - def build_variables(project:, pipeline:) + def build_variables(pipeline:) logger.instrument(:config_build_variables) do - build_variables_without_instrumentation( - project: project, - pipeline: pipeline - ) - end - end - - def build_variables_without_instrumentation(project:, pipeline:) - if use_config_variables? - return pipeline.variables_builder.config_variables - end - - Gitlab::Ci::Variables::Collection.new.tap do |variables| - break variables unless project - - # The order of the following lines is important as priority of CI variables is - # defined globally within GitLab. - # - # See more detail in the docs: https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence - variables.concat(project.predefined_variables) - variables.concat(pipeline.predefined_variables) if pipeline - variables.concat(secret_variables(project: project, pipeline: pipeline)) - variables.concat(project.group.ci_variables_for(source_ref_path, project)) if project.group - variables.concat(project.ci_variables_for(ref: source_ref_path)) - variables.concat(pipeline.variables) if pipeline - variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline&.pipeline_schedule - end - end - - def secret_variables(project:, pipeline:) - if pipeline - pipeline.variables_builder.secret_instance_variables - else - Gitlab::Ci::Variables::Builder::Instance.new.secret_variables + pipeline + .variables_builder + .config_variables end end @@ -195,12 +162,6 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, @context.sentry_payload) end - def use_config_variables? - strong_memoize(:use_config_variables) do - ::Feature.enabled?(:ci_variables_builder_config_variables, @project, default_enabled: :yaml) - end - end - # Overridden in EE def rescue_errors RESCUE_ERRORS diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 2066e9be3b1..bc39abfe977 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -44,7 +44,7 @@ module Gitlab validates :action, type: String, - inclusion: { in: %w[start stop prepare], message: 'should be start, stop or prepare' }, + inclusion: { in: %w[start stop prepare verify access], message: 'should be start, stop, prepare, verify, or access' }, allow_nil: true validates :deployment_tier, diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 06c81fd65dd..7513936a18a 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,7 +11,7 @@ module Gitlab include ::Gitlab::Ci::Config::Entry::Processable ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze - ALLOWED_KEYS = %i[tags script type image services start_in artifacts + ALLOWED_KEYS = %i[tags script image services start_in artifacts cache dependencies before_script after_script environment coverage retry parallel interruptible timeout release].freeze @@ -55,11 +55,6 @@ module Gitlab description: 'Commands that will be executed in this job.', inherit: false - entry :type, Entry::Stage, - description: 'Deprecated: stage this job will be executed into.', - inherit: false, - deprecation: { deprecated: '9.0', warning: '14.8', removed: '15.0' } - entry :after_script, Entry::Commands, description: 'Commands that will be executed when finishing job.', inherit: true @@ -135,19 +130,6 @@ module Gitlab true end - def compose!(deps = nil) - super do - # The type keyword will be removed in 15.0: - # https://gitlab.com/gitlab-org/gitlab/-/issues/346823 - if type_defined? && !stage_defined? - @entries[:stage] = @entries[:type] - log_and_warn_deprecated_entry(@entries[:type]) - end - - @entries.delete(:type) - end - end - def delayed? self.when == 'delayed' end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index f8fce1abc06..d5d204bb995 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -15,7 +15,7 @@ module Gitlab ALLOWED_KEYS = %i[junit codequality sast secret_detection dependency_scanning container_scanning dast performance browser_performance load_performance license_scanning metrics lsif - dotenv cobertura terraform accessibility cluster_applications + dotenv terraform accessibility requirements coverage_fuzzing api_fuzzing cluster_image_scanning coverage_report].freeze @@ -45,14 +45,10 @@ module Gitlab validates :metrics, array_of_strings_or_string: true validates :lsif, array_of_strings_or_string: true validates :dotenv, array_of_strings_or_string: true - validates :cobertura, array_of_strings_or_string: true validates :terraform, array_of_strings_or_string: true validates :accessibility, array_of_strings_or_string: true - validates :cluster_applications, array_of_strings_or_string: true # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441 validates :requirements, array_of_strings_or_string: true end - - validates :config, mutually_exclusive_keys: [:coverage_report, :cobertura] end def value diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 7b58ef0b8ab..ff11c757dfa 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable ALLOWED_KEYS = %i[default include before_script image services - after_script variables stages types cache workflow].freeze + after_script variables stages cache workflow].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -57,11 +57,6 @@ module Gitlab description: 'Configuration of stages for this pipeline.', reserved: true - entry :types, Entry::Stages, - description: 'Deprecated: stages for this pipeline.', - reserved: true, - deprecation: { deprecated: '9.0', warning: '14.8', removed: '15.0' } - entry :cache, Entry::Caches, description: 'Configure caching between build jobs.', reserved: true @@ -100,7 +95,6 @@ module Gitlab def compose!(_deps = nil) super(self) do - compose_deprecated_entries! compose_jobs! end end @@ -118,21 +112,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def compose_deprecated_entries! - ## - # Deprecated `:types` key workaround - if types are defined and - # stages are not defined we use types definition as stages. - # This keyword will be removed in 15.0: - # https://gitlab.com/gitlab-org/gitlab/-/issues/346823 - # - if types_defined? - @entries[:stages] = @entries[:types] unless stages_defined? - log_and_warn_deprecated_entry(@entries[:types]) - end - - @entries.delete(:types) - end - def filter_jobs! return unless @config.is_a?(Hash) diff --git a/lib/gitlab/ci/config/extendable/entry.rb b/lib/gitlab/ci/config/extendable/entry.rb index 0001a259281..169d329fe02 100644 --- a/lib/gitlab/ci/config/extendable/entry.rb +++ b/lib/gitlab/ci/config/extendable/entry.rb @@ -99,7 +99,7 @@ module Gitlab end def circular_dependency? - ancestors.include?(key) + ancestors.include?(key) # rubocop:disable Performance/AncestorsInclude end def unknown_extensions diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index ee9cc1552fe..feb2cbb19ad 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -23,6 +23,8 @@ module Gitlab super.merge( type: :local, location: masked_location, + blob: masked_blob, + raw: masked_raw, extra: {} ) end @@ -57,6 +59,24 @@ module Gitlab variables: context.variables } end + + def masked_blob + strong_memoize(:masked_blob) do + context.mask_variables_from( + Gitlab::Routing.url_helpers.project_blob_url(context.project, ::File.join(context.sha, location)) + ) + end + end + + def masked_raw + return unless context.project + + strong_memoize(:masked_raw) do + context.mask_variables_from( + Gitlab::Routing.url_helpers.project_raw_url(context.project, ::File.join(context.sha, location)) + ) + end + end end end end diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index 3d4436530a8..09c36a1bcb6 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -31,6 +31,8 @@ module Gitlab super.merge( type: :file, location: masked_location, + blob: masked_blob, + raw: masked_raw, extra: { project: masked_project_name, ref: masked_ref_name } ) end @@ -69,6 +71,8 @@ module Gitlab end def sha + return unless project + strong_memoize(:sha) do project.commit(ref_name).try(:sha) end @@ -96,6 +100,26 @@ module Gitlab context.mask_variables_from(ref_name) end end + + def masked_blob + return unless project + + strong_memoize(:masked_blob) do + context.mask_variables_from( + Gitlab::Routing.url_helpers.project_blob_url(project, ::File.join(sha, location)) + ) + end + end + + def masked_raw + return unless project + + strong_memoize(:masked_raw) do + context.mask_variables_from( + Gitlab::Routing.url_helpers.project_raw_url(project, ::File.join(sha, location)) + ) + end + end end end end diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index e7b007b4d8d..7d3a2362246 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -22,6 +22,8 @@ module Gitlab super.merge( type: :remote, location: masked_location, + blob: nil, + raw: masked_location, extra: {} ) end diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index 9469f09ce13..58b81b259cb 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -9,6 +9,7 @@ module Gitlab attr_reader :location SUFFIX = '.gitlab-ci.yml' + HOST = 'https://gitlab.com/gitlab-org/gitlab/-/raw/master' def initialize(params, context) @location = params[:template] @@ -24,6 +25,8 @@ module Gitlab super.merge( type: :template, location: masked_location, + blob: nil, + raw: masked_raw, extra: {} ) end @@ -51,6 +54,14 @@ module Gitlab def fetch_template_content Gitlab::Template::GitlabCiYmlTemplate.find(template_name, context.project)&.content end + + def masked_raw + strong_memoize(:masked_raw) do + context.mask_variables_from( + "#{HOST}/#{Gitlab::Template::GitlabCiYmlTemplate::BASE_DIR}/#{location}" + ) + end + end end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 3fb86b8b3e8..97774bc5e13 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -73,7 +73,7 @@ module Gitlab def key @key ||= begin - key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true) + key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project) Gitlab::CurrentSettings.ci_jwt_signing_key else Rails.application.secrets.openid_connect_signing_key diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 5591ed62436..51743a1f273 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -4,18 +4,23 @@ module Gitlab module Ci class Lint class Result - attr_reader :jobs, :merged_yaml, :errors, :warnings + attr_reader :jobs, :merged_yaml, :errors, :warnings, :includes - def initialize(jobs:, merged_yaml:, errors:, warnings:) + def initialize(jobs:, merged_yaml:, errors:, warnings:, includes:) @jobs = jobs @merged_yaml = merged_yaml @errors = errors @warnings = warnings + @includes = includes end def valid? @errors.empty? end + + def status + valid? ? :valid : :invalid + end end LOG_MAX_DURATION_THRESHOLD = 2.seconds @@ -44,9 +49,10 @@ module Gitlab Result.new( jobs: dry_run_convert_to_jobs(pipeline.stages), - merged_yaml: pipeline.merged_yaml, + merged_yaml: pipeline.config_metadata.try(:[], :merged_yaml), errors: pipeline.error_messages.map(&:content), - warnings: pipeline.warning_messages(limit: ::Gitlab::Ci::Warnings::MAX_LIMIT).map(&:content) + warnings: pipeline.warning_messages(limit: ::Gitlab::Ci::Warnings::MAX_LIMIT).map(&:content), + includes: pipeline.config_metadata.try(:[], :includes) ) end @@ -57,9 +63,10 @@ module Gitlab Result.new( jobs: static_validation_convert_to_jobs(result), - merged_yaml: result.merged_yaml, + merged_yaml: result.config_metadata[:merged_yaml], errors: result.errors, - warnings: result.warnings.take(::Gitlab::Ci::Warnings::MAX_LIMIT) # rubocop: disable CodeReuse/ActiveRecord + warnings: result.warnings.take(::Gitlab::Ci::Warnings::MAX_LIMIT), # rubocop: disable CodeReuse/ActiveRecord + includes: result.config_metadata[:includes] ) ensure logger.commit(pipeline: ::Ci::Pipeline.new, caller: self.class.name) diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index cef029bd749..4460843545e 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -6,39 +6,28 @@ module Gitlab module Security module Validators class SchemaValidator - # https://docs.gitlab.com/ee/update/deprecations.html#147 SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1] + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2] }.freeze - # https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/tags - PREVIOUS_RELEASES = %w[10.0.0 12.0.0 12.1.0 13.0.0 - 13.1.0 2.3.0-rc1 2.3.0-rc1 2.3.1-rc1 2.3.2-rc1 2.3.3-rc1 - 2.4.0-rc1 3.0.0 3.0.0-rc1 3.1.0-rc1 4.0.0-rc1 5.0.0-rc1 - 5.0.1-rc1 6.0.0-rc1 6.0.1-rc1 6.1.0-rc1 7.0.0-rc1 7.0.1-rc1 - 8.0.0-rc1 8.0.1-rc1 8.1.0-rc1 9.0.0-rc1].freeze - - # These come from https://app.periscopedata.com/app/gitlab/895813/Secure-Scan-metrics?widget=12248944&udv=1385516 - KNOWN_VERSIONS_TO_REMOVE = %w[0.1 1.0 1.0.0 1.2 1.3 10.0.0 12.1.0 13.1.0 2.0 2.1 2.1.0 2.3 2.3.0 2.4 3.0 3.0.0 3.0.6 3.13.2 V2.7.0].freeze - - VERSIONS_TO_REMOVE_IN_15_0 = (PREVIOUS_RELEASES + KNOWN_VERSIONS_TO_REMOVE).freeze + VERSIONS_TO_REMOVE_IN_16_0 = [].freeze DEPRECATED_VERSIONS = { - cluster_image_scanning: VERSIONS_TO_REMOVE_IN_15_0, - container_scanning: VERSIONS_TO_REMOVE_IN_15_0, - coverage_fuzzing: VERSIONS_TO_REMOVE_IN_15_0, - dast: VERSIONS_TO_REMOVE_IN_15_0, - api_fuzzing: VERSIONS_TO_REMOVE_IN_15_0, - dependency_scanning: VERSIONS_TO_REMOVE_IN_15_0, - sast: VERSIONS_TO_REMOVE_IN_15_0, - secret_detection: VERSIONS_TO_REMOVE_IN_15_0 + cluster_image_scanning: VERSIONS_TO_REMOVE_IN_16_0, + container_scanning: VERSIONS_TO_REMOVE_IN_16_0, + coverage_fuzzing: VERSIONS_TO_REMOVE_IN_16_0, + dast: VERSIONS_TO_REMOVE_IN_16_0, + api_fuzzing: VERSIONS_TO_REMOVE_IN_16_0, + dependency_scanning: VERSIONS_TO_REMOVE_IN_16_0, + sast: VERSIONS_TO_REMOVE_IN_16_0, + secret_detection: VERSIONS_TO_REMOVE_IN_16_0 }.freeze class Schema @@ -165,7 +154,6 @@ module Gitlab def handle_unsupported_report_version(treat_as:) if report_version.nil? message = "Report version not provided, #{report_type} report type supports versions: #{supported_schema_versions}" - add_message_as(level: treat_as, message: message) else message = "Version #{report_version} for report type #{report_type} is unsupported, supported versions for this report type are: #{supported_schema_versions}" end diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..31840a7e914 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/cluster-image-scanning-report-format.json @@ -0,0 +1,977 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.2" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/container-scanning-report-format.json new file mode 100644 index 00000000000..c70628a0949 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/container-scanning-report-format.json @@ -0,0 +1,911 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.2" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+$", + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..fbc7b4ea733 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/coverage-fuzzing-report-format.json @@ -0,0 +1,874 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.2" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dast-report-format.json new file mode 100644 index 00000000000..3c9db0546b1 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dast-report-format.json @@ -0,0 +1,1287 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.2" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + }, + "discovered_at": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", + "examples": [ + "2020-01-28T03:26:02.956" + ] + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dependency-scanning-report-format.json new file mode 100644 index 00000000000..c7459216faf --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dependency-scanning-report-format.json @@ -0,0 +1,968 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.2" + }, + "required": [ + "dependency_files", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/sast-report-format.json index a7159be0190..20818792652 100644 --- a/lib/gitlab/ci/parsers/security/validators/schemas/sast-report-format.json +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/sast-report-format.json @@ -325,7 +325,7 @@ } }, "self": { - "version": "14.0.0" + "version": "14.1.2" }, "required": [ "version", @@ -384,6 +384,68 @@ } } }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, "scanner": { "type": "object", "description": "Object defining the scanner used to perform the scan.", @@ -426,8 +488,8 @@ ] }, "vendor": { - "type": "object", "description": "The vendor/maintainer of the scanner.", + "type": "object", "required": [ "name" ], @@ -484,7 +546,7 @@ "description": "Array of vulnerability objects.", "items": { "type": "object", - "description": "Describes the vulnerability.", + "description": "Describes the vulnerability using GitLab Flavored Markdown", "required": [ "category", "cve", @@ -629,6 +691,107 @@ "details": { "$ref": "#/definitions/named_list/properties/items" }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, "location": { "type": "object", "description": "Identifies the vulnerability's location.", diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/secret-detection-report-format.json index 462e23a151c..12386d2c1d4 100644 --- a/lib/gitlab/ci/parsers/security/validators/schemas/secret-detection-report-format.json +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/secret-detection-report-format.json @@ -325,7 +325,7 @@ } }, "self": { - "version": "14.0.0" + "version": "14.1.2" }, "required": [ "version", @@ -384,6 +384,68 @@ } } }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, "scanner": { "type": "object", "description": "Object defining the scanner used to perform the scan.", @@ -426,8 +488,8 @@ ] }, "vendor": { - "type": "object", "description": "The vendor/maintainer of the scanner.", + "type": "object", "required": [ "name" ], @@ -484,7 +546,7 @@ "description": "Array of vulnerability objects.", "items": { "type": "object", - "description": "Describes the vulnerability.", + "description": "Describes the vulnerability using GitLab Flavored Markdown", "required": [ "category", "cve", @@ -629,6 +691,107 @@ "details": { "$ref": "#/definitions/named_list/properties/items" }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, "location": { "required": [ "commit" diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/dependency-scanning-report-format.json deleted file mode 120000 index 11e0a6846fb..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/dependency-scanning-report-format.json +++ /dev/null @@ -1 +0,0 @@ -14.0.0/dependency-scanning-report-format.json
\ No newline at end of file diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index c466b8b36d0..0a6f6fd740c 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -96,7 +96,7 @@ module Gitlab step = step_class.name.underscore.parameterize(separator: '_') logger.observe("pipeline_step_#{step}_duration_s", duration) - if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops, default_enabled: :yaml) + if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops) metrics.pipeline_creation_step_duration_histogram .observe({ step: step_class.name }, duration.seconds) end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 64d1b001e3c..5548fca320f 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -35,7 +35,7 @@ module Gitlab error(result.errors.first, config_error: true) end - @pipeline.merged_yaml = result.merged_yaml + @pipeline.config_metadata = result.config_metadata rescue StandardError => ex Gitlab::ErrorTracking.track_exception(ex, diff --git a/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb b/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb index cb02f09f819..17ebf56985b 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb @@ -54,15 +54,13 @@ module Gitlab def throttle_enabled? ::Feature.enabled?( :ci_throttle_pipelines_creation, - project, - default_enabled: :yaml) + project) end def dry_run? ::Feature.enabled?( :ci_throttle_pipelines_creation_dry_run, - project, - default_enabled: :yaml) + project) end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index 4d65b914d8d..6efb3a4f16a 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -11,8 +11,15 @@ module Gitlab def evaluate(variables = {}) text = @left.evaluate(variables) regexp = @right.evaluate(variables) + return false unless regexp + if ::Feature.enabled?(:ci_fix_rules_if_comparison_with_regexp_variable) + # All variables are evaluated as strings, even if they are regexp strings. + # So, we need to convert them to regexp objects. + regexp = Lexeme::Pattern.build_and_evaluate(regexp, variables) + end + regexp.scan(text.to_s).present? end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb index 29c5aa5d753..a72e5dbc822 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb @@ -11,8 +11,15 @@ module Gitlab def evaluate(variables = {}) text = @left.evaluate(variables) regexp = @right.evaluate(variables) + return true unless regexp + if ::Feature.enabled?(:ci_fix_rules_if_comparison_with_regexp_variable) + # All variables are evaluated as strings, even if they are regexp strings. + # So, we need to convert them to regexp objects. + regexp = Lexeme::Pattern.build_and_evaluate(regexp, variables) + end + regexp.scan(text.to_s).empty? end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index c7106f3ec39..cd4106b16bb 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -35,6 +35,18 @@ module Gitlab def self.build(string) new(string) end + + def self.build_and_evaluate(data, variables = {}) + return data if data.is_a?(Gitlab::UntrustedRegexp) + + begin + new_pattern = build(data) + rescue Lexer::SyntaxError + return data + end + + new_pattern.evaluate(variables) + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index e90e764bcd9..798cea34db6 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -8,10 +8,6 @@ module Gitlab class String < Lexeme::Value PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/.freeze - def initialize(value) - super(value) - end - def evaluate(variables = {}) @value.to_s end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb index 6d872fee39d..fa82bbe3275 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb @@ -10,6 +10,8 @@ module Gitlab :value end + attr_reader :value + def initialize(value) @value = value end diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index ee6c3898592..44d905faced 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -110,7 +110,7 @@ module Gitlab def enabled? strong_memoize(:enabled) do - ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops, default_enabled: :yaml) + ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops) end end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index b5e48f210ad..33b9ac9b641 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -46,7 +46,7 @@ module Gitlab name = :gitlab_ci_active_jobs comment = 'Total amount of active jobs' labels = { plan: nil } - buckets = [0, 200, 500, 1_000, 2_000, 5_000, 10_000] + buckets = [0, 200, 500, 1_000, 2_000, 5_000, 10_000, 15_000, 20_000, 30_000, 40_000] ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 54fb1d19ea8..7d8303214a5 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -74,7 +74,7 @@ module Gitlab end def observe_queue_depth(queue, size) - return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics) if !Rails.env.production? && !QUEUE_DEPTH_HISTOGRAMS.include?(queue) raise ArgumentError, "unknown queue depth label: #{queue}" @@ -84,7 +84,7 @@ module Gitlab end def observe_queue_size(size_proc, runner_type) - return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics) size = size_proc.call.to_f self.class.queue_size_total.observe({ runner_type: runner_type }, size) @@ -96,7 +96,7 @@ module Gitlab result = yield - return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics) seconds = ::Gitlab::Metrics::System.monotonic_time - start_time @@ -121,7 +121,7 @@ module Gitlab end def self.observe_active_runners(runners_proc) - return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics) queue_active_runners_total.observe({}, runners_proc.call.to_f) end @@ -250,7 +250,7 @@ module Gitlab end def running_jobs_relation(job) - if ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data, default_enabled: :yaml) + if ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data) ::Ci::RunningBuild.instance_type.where(project_id: job.project_id) else job.project.builds.running.where(runner: ::Ci::Runner.instance_type) diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb index 365864d3317..68c911d3dbb 100644 --- a/lib/gitlab/ci/runner_instructions.rb +++ b/lib/gitlab/ci/runner_instructions.rb @@ -25,7 +25,7 @@ module Gitlab amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-darwin-amd64" }, install_script_template_path: "lib/gitlab/ci/runner_instructions/templates/osx/install.sh", - runner_executable: "sudo gitlab-runner" + runner_executable: "gitlab-runner" }, windows: { human_readable_name: "Windows", diff --git a/lib/gitlab/ci/runner_upgrade_check.rb b/lib/gitlab/ci/runner_upgrade_check.rb index baf041fc358..46b41ed3c6c 100644 --- a/lib/gitlab/ci/runner_upgrade_check.rb +++ b/lib/gitlab/ci/runner_upgrade_check.rb @@ -5,12 +5,19 @@ module Gitlab class RunnerUpgradeCheck include Singleton + STATUSES = { + invalid: 'Runner version is not valid.', + not_available: 'Upgrade is not available for the runner.', + available: 'Upgrade is available for the runner.', + recommended: 'Upgrade is available and recommended for the runner.' + }.freeze + def initialize reset! end def check_runner_upgrade_status(runner_version) - return :unknown unless runner_version + return :invalid unless runner_version releases = RunnerReleases.instance.releases parsed_runner_version = runner_version.is_a?(::Gitlab::VersionInfo) ? runner_version : ::Gitlab::VersionInfo.parse(runner_version) diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index eaa87157716..263fd9d1052 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -16,7 +16,7 @@ module Gitlab def details_path return unless can?(user, :read_pipeline, downstream_pipeline) - if Feature.enabled?(:ci_retry_downstream_pipeline, subject.project, default_enabled: :yaml) + if Feature.enabled?(:ci_retry_downstream_pipeline, subject.project) project_job_path(subject.project, subject) else project_pipeline_path(downstream_project, downstream_pipeline) diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 8020ffee36f..fddcc1492a8 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -178,7 +178,6 @@ include: - template: Jobs/Helm-2to3.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml - - template: Security/Cluster-Image-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index f3d2e293c86..8c63019d743 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.9.1' + AUTO_BUILD_IMAGE_VERSION: 'v1.14.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index f3d2e293c86..8c63019d743 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.9.1' + AUTO_BUILD_IMAGE_VERSION: 'v1.14.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 0cc5090f85e..04b1c4a6f73 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.23.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.25.0' .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index d41182ec9be..5c56594da78 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -12,10 +12,9 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" - DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" - DS_MAJOR_VERSION: 2 + DS_MAJOR_VERSION: 3 dependency_scanning: stage: test @@ -52,6 +51,18 @@ dependency_scanning: paths: - "**/cyclonedx-*.json" +.gemnasium-shared-rule: + exists: + - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' + - '{composer.lock,*/composer.lock,*/*/composer.lock}' + - '{gems.locked,*/gems.locked,*/*/gems.locked}' + - '{go.sum,*/go.sum,*/*/go.sum}' + - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' + - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' + - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' + - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' + - '{conan.lock,*/conan.lock,*/*/conan.lock}' + gemnasium-dependency_scanning: extends: - .ds-analyzer @@ -66,17 +77,20 @@ gemnasium-dependency_scanning: when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/ - exists: - - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' - - '{composer.lock,*/composer.lock,*/*/composer.lock}' - - '{gems.locked,*/gems.locked,*/*/gems.locked}' - - '{go.sum,*/go.sum,*/*/go.sum}' - - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' - - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' - - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' - - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' - - '{conan.lock,*/conan.lock,*/*/conan.lock}' + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-shared-rule, exists] + +.gemnasium-maven-shared-rule: + exists: + - '{build.gradle,*/build.gradle,*/*/build.gradle}' + - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}' + - '{build.sbt,*/build.sbt,*/*/build.sbt}' + - '{pom.xml,*/pom.xml,*/*/pom.xml}' gemnasium-maven-dependency_scanning: extends: @@ -84,9 +98,6 @@ gemnasium-maven-dependency_scanning: - .cyclone-dx-reports variables: DS_ANALYZER_NAME: "gemnasium-maven" - # Stop reporting Gradle as "maven". - # See https://gitlab.com/gitlab-org/gitlab/-/issues/338252 - DS_REPORT_PACKAGE_MANAGER_MAVEN_WHEN_JAVA: "false" rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never @@ -94,12 +105,22 @@ gemnasium-maven-dependency_scanning: when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ - exists: - - '{build.gradle,*/build.gradle,*/*/build.gradle}' - - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}' - - '{build.sbt,*/build.sbt,*/*/build.sbt}' - - '{pom.xml,*/pom.xml,*/*/pom.xml}' + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-maven-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-maven-shared-rule, exists] + +.gemnasium-python-shared-rule: + exists: + - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' + - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' + - '{Pipfile,*/Pipfile,*/*/Pipfile}' + - '{requires.txt,*/requires.txt,*/*/requires.txt}' + - '{setup.py,*/setup.py,*/*/setup.py}' + - '{poetry.lock,*/poetry.lock,*/*/poetry.lock}' gemnasium-python-dependency_scanning: extends: @@ -107,9 +128,6 @@ gemnasium-python-dependency_scanning: - .cyclone-dx-reports variables: DS_ANALYZER_NAME: "gemnasium-python" - # Stop reporting Pipenv and Setuptools as "pip". - # See https://gitlab.com/gitlab-org/gitlab/-/issues/338252 - DS_REPORT_PACKAGE_MANAGER_PIP_WHEN_PYTHON: "false" rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never @@ -117,46 +135,39 @@ gemnasium-python-dependency_scanning: when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ - exists: - - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' - - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' - - '{Pipfile,*/Pipfile,*/*/Pipfile}' - - '{requires.txt,*/requires.txt,*/*/requires.txt}' - - '{setup.py,*/setup.py,*/*/setup.py}' - # Support passing of $PIP_REQUIREMENTS_FILE - # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-python-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-python-shared-rule, exists] + # Support passing of $PIP_REQUIREMENTS_FILE + # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DS_IMAGE_SUFFIX: "-fips" - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ && $PIP_REQUIREMENTS_FILE bundler-audit-dependency_scanning: extends: .ds-analyzer - variables: - DS_ANALYZER_NAME: "bundler-audit" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.0" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/347491" + - exit 1 rules: - - if: $DEPENDENCY_SCANNING_DISABLED - when: never - - if: $DS_EXCLUDED_ANALYZERS =~ /bundler-audit/ - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /bundler-audit/ - exists: - - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' + - when: never retire-js-dependency_scanning: extends: .ds-analyzer - variables: - DS_ANALYZER_NAME: "retire.js" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.0" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/289830" + - exit 1 rules: - - if: $DEPENDENCY_SCANNING_DISABLED - when: never - - if: $DS_EXCLUDED_ANALYZERS =~ /retire.js/ - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /retire.js/ - exists: - - '{package.json,*/package.json,*/*/package.json}' + - when: never diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 89eb91c981f..c29b5b74bfc 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.23.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.25.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 78f28b59aa5..d09bb53a5b1 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.23.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.25.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml index 89a44eddefd..f7945b46a59 100644 --- a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml @@ -14,7 +14,7 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. - LICENSE_MANAGEMENT_VERSION: 3 + LICENSE_MANAGEMENT_VERSION: 4 license_scanning: stage: test diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml new file mode 100644 index 00000000000..b6358eb0831 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml @@ -0,0 +1,41 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/iac_scanning/ +# +# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/iac_scanning/index.html + +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SAST_IMAGE_SUFFIX: "" + + SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" + +iac-sast: + stage: test + artifacts: + reports: + sast: gl-sast-report.json + rules: + - when: never + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + variables: + SEARCH_MAX_DEPTH: 4 + allow_failure: true + script: + - /analyzer run + +kics-iac-sast: + extends: iac-sast + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /kics/ + when: never + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index 488e7ec72fd..b6358eb0831 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -31,7 +31,7 @@ kics-iac-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 1 + SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - if: $SAST_DISABLED diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 7415fa3104c..be41553450c 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -55,7 +55,7 @@ brakeman-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -92,7 +92,7 @@ flawfinder-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -113,7 +113,7 @@ kubesec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -144,7 +144,7 @@ gosec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" mobsf-android-sast: @@ -178,7 +178,7 @@ nodejs-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -194,7 +194,7 @@ phpcs-security-audit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -210,7 +210,7 @@ pmd-apex-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -226,22 +226,14 @@ security-code-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: '2' + SAST_ANALYZER_IMAGE_TAG: '3' SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ when: never - # This rule shim will be removed in %15.0, - # See https://gitlab.com/gitlab-org/gitlab/-/issues/350935 - - if: $CI_COMMIT_BRANCH && $CI_SERVER_VERSION_MAJOR == '14' - exists: - - '**/*.csproj' - - '**/*.vbproj' - if: $CI_COMMIT_BRANCH - variables: - SAST_ANALYZER_IMAGE_TAG: '3' exists: - '**/*.csproj' - '**/*.vbproj' @@ -252,7 +244,7 @@ semgrep-sast: name: "$SAST_ANALYZER_IMAGE" variables: SEARCH_MAX_DEPTH: 20 - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - if: $SAST_DISABLED @@ -275,7 +267,7 @@ sobelow-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -291,7 +283,7 @@ spotbugs-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml new file mode 100644 index 00000000000..f8e6e152ab9 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -0,0 +1,407 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/ +# +# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-variables + +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SAST_IMAGE_SUFFIX: "" + + SAST_EXCLUDED_ANALYZERS: "" + SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" + SCAN_KUBERNETES_MANIFESTS: "false" + +sast: + stage: test + artifacts: + reports: + sast: gl-sast-report.json + rules: + - when: never + variables: + SEARCH_MAX_DEPTH: 4 + script: + - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" + - exit 1 + +.sast-analyzer: + extends: sast + allow_failure: true + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + script: + - /analyzer run + +bandit-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.py' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.py' + +brakeman-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.rb' + - '**/Gemfile' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.rb' + - '**/Gemfile' + +eslint-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.html' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.html' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + +flawfinder-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.c' + - '**/*.cc' + - '**/*.cpp' + - '**/*.c++' + - '**/*.cp' + - '**/*.cxx' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.c' + - '**/*.cc' + - '**/*.cpp' + - '**/*.c++' + - '**/*.cp' + - '**/*.cxx' + +kubesec-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ + when: never + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_MERGE_REQUEST_IID && + $SCAN_KUBERNETES_MANIFESTS == 'true' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + # If there's no open merge request, add it to a *branch* pipeline instead. + - if: $CI_COMMIT_BRANCH && + $SCAN_KUBERNETES_MANIFESTS == 'true' + +gosec-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.go' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.go' + +.mobsf-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" + +mobsf-android-sast: + extends: .mobsf-sast + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_MERGE_REQUEST_IID && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.apk' + - '**/AndroidManifest.xml' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + # If there's no open merge request, add it to a *branch* pipeline instead. + - if: $CI_COMMIT_BRANCH && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.apk' + - '**/AndroidManifest.xml' + +mobsf-ios-sast: + extends: .mobsf-sast + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_MERGE_REQUEST_IID && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.ipa' + - '**/*.xcodeproj/*' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + # If there's no open merge request, add it to a *branch* pipeline instead. + - if: $CI_COMMIT_BRANCH && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.ipa' + - '**/*.xcodeproj/*' + +nodejs-scan-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/package.json' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/package.json' + +phpcs-security-audit-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.php' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.php' + +pmd-apex-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.cls' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.cls' + +security-code-scan-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.csproj' + - '**/*.vbproj' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.csproj' + - '**/*.vbproj' + +semgrep-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SERACH_MAX_DEPTH: 20 + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.py' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + - '**/*.c' + - '**/*.go' + - '**/*.java' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.py' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + - '**/*.c' + - '**/*.go' + - '**/*.java' + +sobelow-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - 'mix.exs' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - 'mix.exs' + +spotbugs-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ + when: never + - if: $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/AndroidManifest.xml' + when: never + - if: $SAST_DISABLED + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.groovy' + - '**/*.java' + - '**/*.scala' + - '**/*.kt' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.groovy' + - '**/*.java' + - '**/*.scala' + - '**/*.kt' diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index 6aacd082fd7..3f18237a525 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SECRET_DETECTION_IMAGE_SUFFIX: "" - SECRETS_ANALYZER_VERSION: "3" + SECRETS_ANALYZER_VERSION: "4" SECRET_DETECTION_EXCLUDED_PATHS: "" .secret-analyzer: @@ -31,37 +31,4 @@ secret_detection: when: never - if: $CI_COMMIT_BRANCH script: - - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - # Historic scan - - if [ "$SECRET_DETECTION_HISTORIC_SCAN" == "true" ]; then echo "Running Secret Detection Historic Scan"; /analyzer run; exit; fi - # Default branch scan - - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit; fi - # Push event - - | - if [ "$CI_COMMIT_BEFORE_SHA" == "0000000000000000000000000000000000000000" ]; - then - # first commit on a new branch - echo ${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt - git fetch --depth=2 origin $CI_COMMIT_REF_NAME - else - # determine commit range so that we can fetch the appropriate depth - # check the exit code to determine if we need to limit the commit_list.txt to CI_COMMIT_SHA. - if ! git log --pretty=format:"%H" ${CI_COMMIT_BEFORE_SHA}..${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt; - then - echo "unable to determine commit range, limiting to ${CI_COMMIT_SHA}" - echo ${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt - else - # append newline to to list since `git log` does not end with a - # newline, this is to keep the log messages consistent - echo >> ${CI_COMMIT_SHA}_commit_list.txt - fi - - # we need to extend the git fetch depth to the number of commits + 1 for the following reasons: - # to include the parent commit of the base commit in this MR/Push event. This is needed because - # `git diff -p` needs something to compare changes in that commit against - git fetch --depth=$(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 1)) origin $CI_COMMIT_REF_NAME - fi - echo "scanning $(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt))) commits for a push event" - export SECRET_DETECTION_COMMITS_FILE=${CI_COMMIT_SHA}_commit_list.txt - /analyzer run - - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml new file mode 100644 index 00000000000..e81e06d1a1d --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml @@ -0,0 +1,36 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection +# +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECRET_DETECTION_IMAGE_SUFFIX: "" + SECRETS_ANALYZER_VERSION: "4" + SECRET_DETECTION_EXCLUDED_PATHS: "" + +.secret-analyzer: + stage: test + image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION$SECRET_DETECTION_IMAGE_SUFFIX" + services: [] + allow_failure: true + variables: + GIT_DEPTH: "50" + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + artifacts: + reports: + secret_detection: gl-secret-detection-report.json + +secret_detection: + extends: .secret-analyzer + rules: + - if: $SECRET_DETECTION_DISABLED + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + script: + - /analyzer run diff --git a/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml b/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml index 67c69115948..64a063388b2 100644 --- a/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml @@ -6,7 +6,7 @@ # Use this template to run MATLAB and Simulink as part of your CI/CD pipeline. The template has three jobs: # - `command`: Run MATLAB scripts, functions, and statements. # - `test`: Run tests authored using the MATLAB unit testing framework or Simulink Test. -# - `test_artifacts_job`: Run MATLAB and Simulink tests, and generate test and coverage artifacts. +# - `test_artifacts`: Run MATLAB and Simulink tests, and generate test and coverage artifacts. # # You can copy and paste one or more jobs in this template into your `.gitlab-ci.yml` file. # You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. @@ -17,20 +17,20 @@ # - The jobs in this template use the `matlab -batch` syntax to start MATLAB. The `-batch` option is supported # in MATLAB R2019a and later. -# The `command` runs MATLAB scripts, functions, and statements. To use the job in your pipeline, -# substitute `command` with the code you want to run. +# The `command` job runs MATLAB scripts, functions, and statements. To use the job in your pipeline, +# substitute `mycommand` with the code you want to run. # command: - script: matlab -batch command + script: matlab -batch mycommand -# If the value of `command` is the name of a MATLAB script or function, do not specify the file extension. -# For example, to run a script named `myscript.m` in the root of your repository, specify the `command` like this: +# If the value of `mycommand` is the name of a MATLAB script or function, do not specify the file extension. +# For example, to run a script named `myscript.m` in the root of your repository, specify `mycommand` like this: # # "myscript" # # If you specify more than one script, function, or statement, use a comma or semicolon to separate them. # For example, to run `myscript.m` in a folder named `myfolder` located in the root of the repository, -# you can specify the `command` like this: +# you can specify `mycommand` like this: # # "addpath('myfolder'), myscript" # @@ -41,7 +41,7 @@ command: # [1] https://www.mathworks.com/help/matlab/ref/assert.html # [2] https://www.mathworks.com/help/matlab/ref/error.html -# The `test` runs the MATLAB and Simulink tests in your project. It calls the [`runtests`][3] function +# The `test` job runs the MATLAB and Simulink tests in your project. It calls the [`runtests`][3] function # to run the tests and then the [`assertSuccess`][4] method to fail the job if any of the tests fail. # test: @@ -55,12 +55,12 @@ test: # [4] https://www.mathworks.com/help/matlab/ref/matlab.unittest.testresult.assertsuccess.html # [5] https://www.mathworks.com/help/matlab/projects.html -# The `test_artifacts_job` runs your tests and additionally generates test and coverage artifacts. +# The `test_artifacts` job runs your tests and additionally generates test and coverage artifacts. # It uses the plugin classes in the [`matlab.unittest.plugins`][6] package to generate a JUnit test results -# report and a Cobertura code coverage report. Like the `run_tests` job, this job runs all the tests in your +# report and a Cobertura code coverage report. Like the `test` job, this job runs all the tests in your # project and fails the build if any of the tests fail. # -test_artifacts_job: +test_artifacts: script: | matlab -batch " import matlab.unittest.TestRunner @@ -84,11 +84,13 @@ test_artifacts_job: artifacts: reports: junit: "./artifacts/results.xml" - cobertura: "./artifacts/cobertura.xml" + coverage_report: + coverage_format: cobertura + path: "./artifacts/cobertura.xml" paths: - "./artifacts" -# You can modify the contents of the `test_artifacts_job` depending on your goals. For more +# You can modify the contents of the `test_artifacts` job depending on your goals. For more # information on how to customize the test runner and generate various test and coverage artifacts, # see [Generate Artifacts Using MATLAB Unit Test Plugins][7]. # diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml deleted file mode 100644 index ca63e942130..00000000000 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ /dev/null @@ -1,32 +0,0 @@ -################################################################################ -# WARNING -################################################################################ -# -# This template is DEPRECATED and scheduled for removal in GitLab 15.0 -# See https://gitlab.com/gitlab-org/gitlab/-/issues/333610 for more context. -# -# To get started with a Cluster Management Project, we instead recommend -# using the updated project template: -# -# - Documentation: https://docs.gitlab.com/ee/user/clusters/management_project_template.html -# - Source code: https://gitlab.com/gitlab-org/project-templates/cluster-management/ -# -################################################################################ - -apply: - stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.43.1" - environment: - name: production - variables: - TILLER_NAMESPACE: gitlab-managed-apps - GITLAB_MANAGED_APPS_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/config.yaml - script: - - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml - only: - variables: - - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - artifacts: - when: on_failure - paths: - - tiller.log diff --git a/lib/gitlab/ci/templates/Qualys-IaC-Security.gitlab-ci.yml b/lib/gitlab/ci/templates/Qualys-IaC-Security.gitlab-ci.yml index 6dbd0ce9561..60707dd0df0 100644 --- a/lib/gitlab/ci/templates/Qualys-IaC-Security.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Qualys-IaC-Security.gitlab-ci.yml @@ -3,7 +3,8 @@ # # This template shows how to use Qualys IaC Scan with a GitLab CI/CD pipeline. # Qualys and GitLab users can use this to scan their IaC templates for misconfigurations. -# Documentation about this integration: https://www.qualys.com/documentation/qualys-iac-gitlab-integration.pdf +# The IaC templates are uploaded to Qualys Platform for scanning, which returns the results to GitLab for reporting. +# Documentation about this integration: https://www.qualys.com/docs/qualys-iac-security-integration-gitlab.pdf # # This template should not need editing to work in your project. # It is not designed to be included in an existing CI/CD configuration with the "include:" keyword. diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 33c0928db6f..44f959468a8 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -29,7 +29,7 @@ before_script: - ruby -v # Print out ruby version for debugging # Uncomment next line if your rails app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby + - bundle config set --local deployment true # Install dependencies into ./vendor/ruby - bundle install -j $(nproc) # Optional - Delete if not using `rubocop` diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index aff8b6cb7fa..2fd5b409f5e 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -3,19 +3,36 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: API-Fuzzing.gitlab-ci.yml +# +# You also need to add a `fuzz` stage to your `stages:` configuration. A sample configuration for API Fuzzing: +# +# stages: +# - build +# - test +# - deploy +# - fuzz + # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ # -# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# Configure API Fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: - FUZZAPI_VERSION: "1" + # Setting this variable affects all Security templates + # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" - FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} + # + FUZZAPI_VERSION: "2" + FUZZAPI_IMAGE_SUFFIX: "" + FUZZAPI_IMAGE: api-security apifuzzer_fuzz: stage: fuzz - image: $FUZZAPI_IMAGE + image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION$FUZZAPI_IMAGE_SUFFIX allow_failure: true rules: - if: $API_FUZZING_DISABLED @@ -23,6 +40,10 @@ apifuzzer_fuzz: - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" + variables: + FUZZAPI_IMAGE_SUFFIX: "-fips" - if: $CI_COMMIT_BRANCH script: - /peach/analyzer-fuzz-api diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index b6e811aa84f..450969fcdab 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -26,9 +26,9 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # - FUZZAPI_VERSION: "1" + FUZZAPI_VERSION: "2" FUZZAPI_IMAGE_SUFFIX: "" - FUZZAPI_IMAGE: api-fuzzing + FUZZAPI_IMAGE: api-security apifuzzer_fuzz: stage: fuzz diff --git a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml deleted file mode 100644 index 6b861510eef..00000000000 --- a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Use this template to enable cluster image scanning in your project. -# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` -# keyword. -# The template should work without modifications but you can customize the template settings if -# needed: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#customize-the-container-scanning-settings -# -# Requirements: -# - A `test` stage to be present in the pipeline. -# - You must define the `CIS_KUBECONFIG` variable to allow analyzer to connect to your Kubernetes cluster and fetch found vulnerabilities. -# -# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). -# List of available variables: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#available-variables - -variables: - CIS_ANALYZER_IMAGE: registry.gitlab.com/security-products/cluster-image-scanning:0 - -cluster_image_scanning: - image: "$CIS_ANALYZER_IMAGE" - stage: test - allow_failure: true - artifacts: - reports: - cluster_image_scanning: gl-cluster-image-scanning-report.json - paths: [gl-cluster-image-scanning-report.json] - dependencies: [] - script: - - /analyzer run - rules: - - if: $CLUSTER_IMAGE_SCANNING_DISABLED - when: never - - if: '($KUBECONFIG == null || $KUBECONFIG == "") && ($CIS_KUBECONFIG == null || $CIS_KUBECONFIG == "")' - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcluster_image_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 66db311f897..bec269e2933 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -22,7 +22,7 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: - CS_ANALYZER_IMAGE: registry.gitlab.com/security-products/container-scanning:4 + CS_ANALYZER_IMAGE: registry.gitlab.com/security-products/container-scanning:5 container_scanning: image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" @@ -47,10 +47,8 @@ container_scanning: - if: $CONTAINER_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && $CI_GITLAB_FIPS_MODE == "true" && $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ variables: CS_IMAGE_SUFFIX: -fips - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml index d82f9f06f8d..893098d33c4 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -26,12 +26,13 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # - DAST_API_VERSION: "1" - DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION + DAST_API_VERSION: "2" + DAST_API_IMAGE_SUFFIX: "" + DAST_API_IMAGE: api-security dast_api: stage: dast - image: $DAST_API_IMAGE + image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION$DAST_API_IMAGE_SUFFIX allow_failure: true rules: - if: $DAST_API_DISABLED @@ -39,6 +40,10 @@ dast_api: - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_API_IMAGE_SUFFIX: "-fips" - if: $CI_COMMIT_BRANCH script: - /peach/analyzer-dast-api @@ -50,3 +55,5 @@ dast_api: - gl-*.log reports: dast: gl-dast-api-report.json + +# end diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml index b491b3e3c0c..3acc3b06031 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -1,7 +1,7 @@ # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dast-API.gitlab-ci.yml +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dast-API.latest.gitlab-ci.yml # To use this template, add the following to your .gitlab-ci.yml file: # @@ -26,9 +26,9 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # - DAST_API_VERSION: "1" + DAST_API_VERSION: "2" DAST_API_IMAGE_SUFFIX: "" - DAST_API_IMAGE: api-fuzzing + DAST_API_IMAGE: api-security dast_api: stage: dast diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml index 998425aa141..c71a1b1873a 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml @@ -10,7 +10,7 @@ stages: - dast variables: - DAST_VERSION: 2 + DAST_VERSION: 3 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index e8e7fe62e70..3bc44fe5e1b 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -22,7 +22,7 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: - DAST_VERSION: 2 + DAST_VERSION: 3 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index c755211ec11..e5ac5099546 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -22,7 +22,7 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: - DAST_VERSION: 2 + DAST_VERSION: 3 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" diff --git a/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml new file mode 100644 index 00000000000..2207d4ec17a --- /dev/null +++ b/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml @@ -0,0 +1,2 @@ +include: + template: Jobs/SAST-IaC.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index a6fd070ec34..b34bfe2a53c 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -18,8 +18,7 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SECURE_BINARIES_ANALYZERS: >- - bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kics, kubesec, semgrep, - bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, + bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kics, kubesec, semgrep, gemnasium, gemnasium-maven, gemnasium-python, license-finder, dast, dast-runner-validation, api-fuzzing @@ -68,6 +67,8 @@ variables: bandit: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "2" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -75,6 +76,8 @@ bandit: brakeman: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -91,6 +94,8 @@ gosec: spotbugs: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -98,6 +103,8 @@ spotbugs: flawfinder: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -105,6 +112,8 @@ flawfinder: phpcs-security-audit: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -121,6 +130,8 @@ security-code-scan: nodejs-scan: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -128,6 +139,8 @@ nodejs-scan: eslint: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "2" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -140,10 +153,12 @@ secrets: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/ variables: - SECURE_BINARIES_ANALYZER_VERSION: "3" + SECURE_BINARIES_ANALYZER_VERSION: "4" semgrep: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -151,6 +166,8 @@ semgrep: sobelow: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -158,6 +175,8 @@ sobelow: pmd-apex: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -165,6 +184,8 @@ pmd-apex: kubesec: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -174,20 +195,6 @@ kubesec: # Dependency Scanning jobs # -bundler-audit: - extends: .download_images - only: - variables: - - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && - $SECURE_BINARIES_ANALYZERS =~ /\bbundler-audit\b/ - -retire.js: - extends: .download_images - only: - variables: - - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && - $SECURE_BINARIES_ANALYZERS =~ /\bretire\.js\b/ - gemnasium: extends: .download_images only: diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml deleted file mode 100644 index 55648437191..00000000000 --- a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml +++ /dev/null @@ -1,35 +0,0 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml - -# GitLab Serverless template - -image: alpine:latest - -stages: - - build - - test - - deploy - -.serverless:build:image: - image: registry.gitlab.com/gitlab-org/gitlabktl:latest - stage: build - script: /usr/bin/gitlabktl app build - -.serverless:deploy:image: - image: registry.gitlab.com/gitlab-org/gitlabktl:latest - stage: deploy - environment: development - script: /usr/bin/gitlabktl app deploy - -.serverless:build:functions: - image: registry.gitlab.com/gitlab-org/gitlabktl:latest - stage: build - script: /usr/bin/gitlabktl serverless build - -.serverless:deploy:functions: - image: registry.gitlab.com/gitlab-org/gitlabktl:latest - stage: deploy - environment: development - script: /usr/bin/gitlabktl serverless deploy diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 1a857ef3eb3..56151a6bcdf 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -1,27 +1,32 @@ # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml include: - - template: Terraform/Base.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml + - template: Terraform/Base.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml + - template: Jobs/SAST-IaC.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml stages: - - init - validate + - test - build - deploy -init: - extends: .init +fmt: + extends: .terraform:fmt + needs: [] validate: - extends: .validate + extends: .terraform:validate + needs: [] build: - extends: .build + extends: .terraform:build deploy: - extends: .deploy + extends: .terraform:deploy dependencies: - build + environment: + name: $TF_STATE_NAME diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index 12c987a8d37..019b970bc30 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -5,7 +5,7 @@ include: - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml - - template: Jobs/SAST-IaC.latest.gitlab-ci.yml + - template: Jobs/SAST-IaC.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml stages: - validate diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index 84a962e1541..49bdd4b7713 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Terraform/Base.latest +# Terraform/Base # # The purpose of this template is to provide flexibility to the user so # they are able to only include the jobs that they find interesting. @@ -7,10 +7,9 @@ # create hidden jobs. See: https://docs.gitlab.com/ee/ci/yaml/#hide-jobs # # There is a more opinionated template which we suggest the users to abide, -# which is the lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml - +# which is the lib/gitlab/ci/templates/Terraform.gitlab-ci.yml image: - name: registry.gitlab.com/gitlab-org/terraform-images/releases/terraform:1.0.3 + name: registry.gitlab.com/gitlab-org/terraform-images/releases/terraform:1.1.9 variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project @@ -21,43 +20,46 @@ cache: paths: - ${TF_ROOT}/.terraform/ -.init: &init - stage: init +.terraform:fmt: &terraform_fmt + stage: validate script: - cd "${TF_ROOT}" - - gitlab-terraform init + - gitlab-terraform fmt + allow_failure: true -.validate: &validate +.terraform:validate: &terraform_validate stage: validate script: - cd "${TF_ROOT}" - gitlab-terraform validate -.build: &build +.terraform:build: &terraform_build stage: build script: - cd "${TF_ROOT}" - gitlab-terraform plan - gitlab-terraform plan-json + resource_group: ${TF_STATE_NAME} artifacts: paths: - ${TF_ROOT}/plan.cache reports: terraform: ${TF_ROOT}/plan.json -.deploy: &deploy +.terraform:deploy: &terraform_deploy stage: deploy script: - cd "${TF_ROOT}" - gitlab-terraform apply - when: manual - only: - variables: - - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + resource_group: ${TF_STATE_NAME} + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual -.destroy: &destroy +.terraform:destroy: &terraform_destroy stage: cleanup script: - cd "${TF_ROOT}" - gitlab-terraform destroy + resource_group: ${TF_STATE_NAME} when: manual diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index a0ec07e61e1..9ba009a5bca 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -24,20 +24,20 @@ cache: .terraform:fmt: &terraform_fmt stage: validate script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform fmt allow_failure: true .terraform:validate: &terraform_validate stage: validate script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform validate .terraform:build: &terraform_build stage: build script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform plan - gitlab-terraform plan-json resource_group: ${TF_STATE_NAME} @@ -50,7 +50,7 @@ cache: .terraform:deploy: &terraform_deploy stage: deploy script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform apply resource_group: ${TF_STATE_NAME} rules: @@ -60,7 +60,7 @@ cache: .terraform:destroy: &terraform_destroy stage: cleanup script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform destroy resource_group: ${TF_STATE_NAME} when: manual diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 5ea2bc07ffa..2b5e86f4066 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -13,7 +13,7 @@ stages: a11y: stage: accessibility - image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.1.1 + image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.2.3 script: - /gitlab-accessibility.sh "$a11y_urls" allow_failure: true diff --git a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml index 09fce67db2d..b8d284532bd 100644 --- a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml @@ -1,7 +1,7 @@ # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET-Core.yml +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml # This is a simple example illustrating how to build and test .NET Core project # with GitLab Continuous Integration / Continuous Delivery. diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 7d08f0230fc..e93bd75a9fa 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -74,11 +74,11 @@ module Gitlab end def exist? - archived_trace_exist? || live_trace_exist? + archived? || live_trace_exist? end - def archived_trace_exist? - archived? + def archived? + trace_artifact&.stored? end def live_trace_exist? @@ -218,12 +218,6 @@ module Gitlab end end - def archived? - # TODO check checksum to ensure archive completed successfully - # See https://gitlab.com/gitlab-org/gitlab/-/issues/259619 - trace_artifact&.archived_trace_exists? - end - def destroy_any_orphan_trace_data! return unless trace_artifact @@ -312,7 +306,7 @@ module Gitlab end def consistent_archived_trace?(build) - ::Feature.enabled?(:gitlab_ci_archived_trace_consistent_reads, build.project, default_enabled: false) + ::Feature.enabled?(:gitlab_ci_archived_trace_consistent_reads, build.project) end def being_watched_cache_key diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index bcb1fe83ea2..a452cb197ae 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -52,7 +52,7 @@ module Gitlab # https://gitlab.com/groups/gitlab-org/configure/-/epics/8 # Until then, we need to make both the old and the new KUBECONFIG contexts available collection.concat(deployment_variables(environment: environment, job: job)) - template = ::Ci::GenerateKubeconfigService.new(job).execute + template = ::Ci::GenerateKubeconfigService.new(pipeline, token: job.token).execute kubeconfig_yaml = collection['KUBECONFIG']&.value template.merge_yaml(kubeconfig_yaml) if kubeconfig_yaml.present? diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index f14279dca2d..576fb509d47 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -103,10 +103,6 @@ module Gitlab }.compact }.compact end - def merged_yaml - @ci_config&.to_hash&.deep_stringify_keys&.to_yaml - end - def variables_with_data @ci_config.variables_with_data end @@ -127,6 +123,10 @@ module Gitlab jobs.dig(job_name, :stage) end + def config_metadata + @ci_config&.metadata || {} + end + private def variables diff --git a/lib/gitlab/color.rb b/lib/gitlab/color.rb index e0caabb0ec6..01c534c15a0 100644 --- a/lib/gitlab/color.rb +++ b/lib/gitlab/color.rb @@ -170,6 +170,11 @@ module Gitlab Constants::COLOR_NAME_TO_HEX[color.downcase] || new(color) end + # Generate a hex color based on hex-encoded value + def self.color_for(value) + Color.new("##{Digest::SHA256.hexdigest(value.to_s)[0..5]}") + end + def to_s @value.to_s end diff --git a/lib/gitlab/config/entry/validator.rb b/lib/gitlab/config/entry/validator.rb index e5efd4a7b0a..297645a65c1 100644 --- a/lib/gitlab/config/entry/validator.rb +++ b/lib/gitlab/config/entry/validator.rb @@ -7,10 +7,6 @@ module Gitlab include ActiveModel::Validations include Entry::Validators - def initialize(entry) - super(entry) - end - def messages errors.full_messages.map do |error| "#{location} #{error}".downcase diff --git a/lib/gitlab/config/loader/yaml.rb b/lib/gitlab/config/loader/yaml.rb index f3a3818f010..0559c85647d 100644 --- a/lib/gitlab/config/loader/yaml.rb +++ b/lib/gitlab/config/loader/yaml.rb @@ -41,7 +41,7 @@ module Gitlab end def too_big? - return false unless Feature.enabled?(:ci_yaml_limit_size, default_enabled: true) + return false unless Feature.enabled?(:ci_yaml_limit_size) !deep_size.valid? end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 22a4ba8ac7a..521dec110a8 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -61,7 +61,9 @@ module Gitlab end def initialize(csp_directives) - @csp_directives = HashWithIndifferentAccess.new(csp_directives) + @merged_csp_directives = + HashWithIndifferentAccess.new(csp_directives) + .reverse_merge(::Gitlab::ContentSecurityPolicy::ConfigLoader.default_directives) end def load(policy) @@ -77,8 +79,9 @@ module Gitlab private def arguments_for(directive) - arguments = @csp_directives[directive.to_s] - + # In order to disable a directive, the user can explicitly + # set a falsy value like nil, false or empty string + arguments = @merged_csp_directives[directive] return unless arguments.present? && arguments.is_a?(String) arguments.strip.split(' ').map(&:strip) diff --git a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb index 2b1529bdc1a..83ff61bbef2 100644 --- a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb +++ b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb @@ -21,7 +21,7 @@ module Gitlab end def unit - _('per day') + _('/day') end def links diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/data_builder/issuable.rb index add9e880475..9a0b964915c 100644 --- a/lib/gitlab/hook_data/issuable_builder.rb +++ b/lib/gitlab/data_builder/issuable.rb @@ -1,11 +1,15 @@ # frozen_string_literal: true module Gitlab - module HookData - class IssuableBuilder < BaseBuilder + module DataBuilder + class Issuable CHANGES_KEYS = %i[previous current].freeze - alias_method :issuable, :object + attr_reader :issuable + + def initialize(issuable) + @issuable = issuable + end def build(user: nil, changes: {}) hook_data = { diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 1895f0fab32..677b4485288 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -49,7 +49,7 @@ module Gitlab # It does not include the default public schema EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze - PRIMARY_DATABASE_NAME = ActiveRecord::Base.connection_db_config.name.to_sym + PRIMARY_DATABASE_NAME = ActiveRecord::Base.connection_db_config.name.to_sym # rubocop:disable Database/MultipleDatabases def self.database_base_models @database_base_models ||= { @@ -94,21 +94,6 @@ module Gitlab Gitlab::Application.config.database_configuration[Rails.env].include?(database_name.to_s) end - def self.main_database?(name) - # The database is `main` if it is a first entry in `database.yml` - # Rails internally names them `primary` to avoid confusion - # with broad `primary` usage we use `main` instead - # - # TODO: The explicit `== 'main'` is needed in a transition period till - # the `database.yml` is not migrated into `main:` syntax - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65243 - ActiveRecord::Base.configurations.primary?(name.to_s) || name.to_s == 'main' - end - - def self.ci_database?(name) - name.to_s == CI_DATABASE_NAME - end - class PgUser < ApplicationRecord self.table_name = 'pg_user' self.primary_key = :usename diff --git a/lib/gitlab/database/background_migration/batch_optimizer.rb b/lib/gitlab/database/background_migration/batch_optimizer.rb index 58c4a214077..c8fdf8281cd 100644 --- a/lib/gitlab/database/background_migration/batch_optimizer.rb +++ b/lib/gitlab/database/background_migration/batch_optimizer.rb @@ -41,7 +41,7 @@ module Gitlab end def optimize! - return unless Feature.enabled?(:optimize_batched_migrations, type: :ops, default_enabled: :yaml) + return unless Feature.enabled?(:optimize_batched_migrations, type: :ops) if multiplier = batch_size_multiplier max_batch = migration.max_batch_size || MAX_BATCH_SIZE diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index d94bf060d05..a90cae7aea2 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -28,6 +28,8 @@ module Gitlab # on_hold_until is a temporary runtime status which puts execution "on hold" scope :executable, -> { with_status(:active).where('on_hold_until IS NULL OR on_hold_until < NOW()') } + scope :created_after, ->(time) { where('created_at > ?', time) } + scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do where(job_class_name: job_class_name, table_name: table_name, column_name: column_name) .where("job_arguments = ?", job_arguments.to_json) # rubocop:disable Rails/WhereEquals diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb index ec68f401ca2..5f4b2be3da8 100644 --- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -39,7 +39,40 @@ module Gitlab end def execute_batch(tracking_record) - job_instance = migration_instance_for(tracking_record.migration_job_class) + job_instance = execute_job(tracking_record) + + if job_instance.respond_to?(:batch_metrics) + tracking_record.metrics = job_instance.batch_metrics + end + end + + def execute_job(tracking_record) + job_class = tracking_record.migration_job_class + + if job_class < Gitlab::BackgroundMigration::BatchedMigrationJob + execute_batched_migration_job(job_class, tracking_record) + else + execute_legacy_job(job_class, tracking_record) + end + end + + def execute_batched_migration_job(job_class, tracking_record) + job_instance = job_class.new( + start_id: tracking_record.min_value, + end_id: tracking_record.max_value, + batch_table: tracking_record.migration_table_name, + batch_column: tracking_record.migration_column_name, + sub_batch_size: tracking_record.sub_batch_size, + pause_ms: tracking_record.pause_ms, + connection: connection) + + job_instance.perform(*tracking_record.migration_job_arguments) + + job_instance + end + + def execute_legacy_job(job_class, tracking_record) + job_instance = job_class.new job_instance.perform( tracking_record.min_value, @@ -50,17 +83,7 @@ module Gitlab tracking_record.pause_ms, *tracking_record.migration_job_arguments) - if job_instance.respond_to?(:batch_metrics) - tracking_record.metrics = job_instance.batch_metrics - end - end - - def migration_instance_for(job_class) - if job_class < Gitlab::BackgroundMigration::BaseJob - job_class.new(connection: connection) - else - job_class.new - end + job_instance end end end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index ae0ea919b62..036ce7d7631 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -217,7 +217,6 @@ geo_event_log: :gitlab_main geo_events: :gitlab_main geo_hashed_storage_attachments_events: :gitlab_main geo_hashed_storage_migrated_events: :gitlab_main -geo_job_artifact_deleted_events: :gitlab_main geo_lfs_object_deleted_events: :gitlab_main geo_node_namespace_links: :gitlab_main geo_nodes: :gitlab_main @@ -327,6 +326,7 @@ namespace_aggregation_schedules: :gitlab_main namespace_limits: :gitlab_main namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main +namespace_ci_cd_settings: :gitlab_main namespace_settings: :gitlab_main namespaces: :gitlab_main namespaces_sync_events: :gitlab_main @@ -348,6 +348,7 @@ operations_strategies: :gitlab_main operations_strategies_user_lists: :gitlab_main operations_user_lists: :gitlab_main packages_build_infos: :gitlab_main +packages_cleanup_policies: :gitlab_main packages_composer_cache_files: :gitlab_main packages_composer_metadata: :gitlab_main packages_conan_file_metadata: :gitlab_main @@ -388,6 +389,7 @@ plan_limits: :gitlab_main plans: :gitlab_main pool_repositories: :gitlab_main postgres_async_indexes: :gitlab_shared +postgres_autovacuum_activity: :gitlab_shared postgres_foreign_keys: :gitlab_shared postgres_index_bloat_estimates: :gitlab_shared postgres_indexes: :gitlab_shared diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index 3f03d9e2c12..0ddc745ebae 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -90,7 +90,7 @@ module Gitlab return false unless ::Gitlab::SafeRequestStore.active? ::Gitlab::SafeRequestStore.fetch(:force_no_sharing_primary_model) do - ::Feature::FlipperFeature.table_exists? && ::Feature.enabled?(:force_no_sharing_primary_model, default_enabled: :yaml) + ::Feature::FlipperFeature.table_exists? && ::Feature.enabled?(:force_no_sharing_primary_model) end end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 1e27bcfc55d..191ebe18b8a 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -255,6 +255,7 @@ module Gitlab # ActiveRecord::ConnectionAdapters::ConnectionHandler handles fetching, # and caching for connections pools for each "connection", so we # leverage that. + # rubocop:disable Database/MultipleDatabases def pool ActiveRecord::Base.connection_handler.retrieve_connection_pool( @configuration.primary_connection_specification_name, @@ -262,6 +263,7 @@ module Gitlab shard: ActiveRecord::Base.default_shard ) || raise(::ActiveRecord::ConnectionNotEstablished) end + # rubocop:enable Database/MultipleDatabases def wal_diff(location1, location2) read_write do |connection| diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb index dc695a74a4b..038af570dbc 100644 --- a/lib/gitlab/database/migration.rb +++ b/lib/gitlab/database/migration.rb @@ -37,18 +37,19 @@ module Gitlab class V1_0 < ActiveRecord::Migration[6.1] # rubocop:disable Naming/ClassAndModuleCamelCase include LockRetriesConcern include Gitlab::Database::MigrationHelpers::V2 - end - - class V2_0 < V1_0 # rubocop:disable Naming/ClassAndModuleCamelCase - include Gitlab::Database::MigrationHelpers::RestrictGitlabSchema # When running migrations, the `db:migrate` switches connection of # ActiveRecord::Base depending where the migration runs. # This helper class is provided to avoid confusion using `ActiveRecord::Base` class MigrationRecord < ActiveRecord::Base + self.abstract_class = true # Prevent STI behavior end end + class V2_0 < V1_0 # rubocop:disable Naming/ClassAndModuleCamelCase + include Gitlab::Database::MigrationHelpers::RestrictGitlabSchema + end + def self.[](version) version = version.to_s name = "V#{version.tr('.', '_')}" diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index d016dea224b..0453b81d67d 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -3,6 +3,7 @@ module Gitlab module Database module MigrationHelpers + include Migrations::ReestablishedConnectionStack include Migrations::BackgroundMigrationHelpers include Migrations::BatchedBackgroundMigrationHelpers include DynamicModelHelpers @@ -943,7 +944,7 @@ module Gitlab execute("DELETE FROM batched_background_migrations WHERE #{conditions}") end - def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:) + def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:, finalize: true) migration = Gitlab::Database::BackgroundMigration::BatchedMigration .for_configuration(job_class_name, table_name, column_name, job_arguments).first @@ -954,14 +955,18 @@ module Gitlab job_arguments: job_arguments } - if migration.nil? - Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" - elsif !migration.finished? + return Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" if migration.nil? + + return if migration.finished? + + finalize_batched_background_migration(job_class_name: job_class_name, table_name: table_name, column_name: column_name, job_arguments: job_arguments) if finalize + + unless migration.reload.finished? # rubocop:disable Cop/ActiveRecordAssociationReload raise "Expected batched background migration for the given configuration to be marked as 'finished', " \ "but it is '#{migration.status_name}':" \ "\t#{configuration}" \ "\n\n" \ - "Finalize it manualy by running" \ + "Finalize it manually by running" \ "\n\n" \ "\tsudo gitlab-rake gitlab:background_migrations:finalize[#{job_class_name},#{table_name},#{column_name},'#{job_arguments.to_json.gsub(',', '\,')}']" \ "\n\n" \ diff --git a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb index 5a25128f3a9..d8d07fcaf2d 100644 --- a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb +++ b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb @@ -27,7 +27,7 @@ module Gitlab return end - Gitlab::Database::QueryAnalyzer.instance.within([validator_class]) do + Gitlab::Database::QueryAnalyzer.instance.within([validator_class, connection_validator_class]) do validator_class.allowed_gitlab_schemas = self.allowed_gitlab_schemas super @@ -45,6 +45,10 @@ module Gitlab Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas end + def connection_validator_class + Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection + end + def unmatched_schemas (self.allowed_gitlab_schemas || []) - allowed_schemas_for_connection end diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 7e5c002d072..9bffed43077 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -41,6 +41,25 @@ module Gitlab # end # end def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BATCH_SIZE, other_job_arguments: [], initial_delay: 0, track_jobs: false, primary_column_name: :id) + if transaction_open? + raise 'The `#queue_background_migration_jobs_by_range_at_intervals` can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + # Background Migrations do not work well for in cases requiring to update `gitlab_shared` + # Once the decomposition is done, enqueued jobs for `gitlab_shared` tables (on CI database) + # will not be executed since the queue (which is stored in Redis) is tied to main database, not to schema. + # The batched background migrations do not have those limitations since the tracking tables + # are properly database-only. + if background_migration_restrict_gitlab_migration_schemas&.include?(:gitlab_shared) + raise 'The `#queue_background_migration_jobs_by_range_at_intervals` cannot " \ + "use `restrict_gitlab_migration:` " with `:gitlab_shared`. ' \ + 'Background migrations do encode migration worker which is tied to a given database. ' \ + 'After split this worker will not be properly duplicated into decomposed database. ' \ + 'Use batched background migrations instead that do support well working across all databases.' + end + raise "#{model_class} does not have an ID column of #{primary_column_name} to use for batch ranges" unless model_class.column_names.include?(primary_column_name.to_s) raise "#{primary_column_name} is not an integer or string column" unless [:integer, :string].include?(model_class.columns_hash[primary_column_name.to_s].type) @@ -90,6 +109,18 @@ module Gitlab # delay_interval - The duration between each job's scheduled time # batch_size - The maximum number of jobs to fetch to memory from the database. def requeue_background_migration_jobs_by_range_at_intervals(job_class_name, delay_interval, batch_size: BATCH_SIZE, initial_delay: 0) + if transaction_open? + raise 'The `#requeue_background_migration_jobs_by_range_at_intervals` can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if background_migration_restrict_gitlab_migration_schemas&.any? + raise 'The `#requeue_background_migration_jobs_by_range_at_intervals` cannot use `restrict_gitlab_migration:`. ' \ + 'The `#requeue_background_migration_jobs_by_range_at_intervals` needs to be executed on all databases since ' \ + 'each database has its own queue of background migrations.' + end + job_coordinator = coordinator_for_tracking_database # To not overload the worker too much we enforce a minimum interval both @@ -133,23 +164,40 @@ module Gitlab # This method does not garauntee that all jobs completed successfully. # It can only be used if the previous background migration used the queue_background_migration_jobs_by_range_at_intervals helper. def finalize_background_migration(class_name, delete_tracking_jobs: ['succeeded']) + if transaction_open? + raise 'The `#finalize_background_migration` can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if background_migration_restrict_gitlab_migration_schemas&.any? + raise 'The `#finalize_background_migration` cannot use `restrict_gitlab_migration:`. ' \ + 'The `#finalize_background_migration` needs to be executed on all databases since ' \ + 'each database has its own queue of background migrations.' + end + job_coordinator = coordinator_for_tracking_database - # Empty the sidekiq queue. - job_coordinator.steal(class_name) + with_restored_connection_stack do + # Since we are running trusted code (background migration class) allow to execute any type of finalize + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + # Empty the sidekiq queue. + job_coordinator.steal(class_name) - # Process pending tracked jobs. - jobs = Gitlab::Database::BackgroundMigrationJob.pending.for_migration_class(class_name) + # Process pending tracked jobs. + jobs = Gitlab::Database::BackgroundMigrationJob.pending.for_migration_class(class_name) - jobs.find_each do |job| - job_coordinator.perform(job.class_name, job.arguments) - end + jobs.find_each do |job| + job_coordinator.perform(job.class_name, job.arguments) + end - # Empty the sidekiq queue. - job_coordinator.steal(class_name) + # Empty the sidekiq queue. + job_coordinator.steal(class_name) - # Delete job tracking rows. - delete_job_tracking(class_name, status: delete_tracking_jobs) if delete_tracking_jobs + # Delete job tracking rows. + delete_job_tracking(class_name, status: delete_tracking_jobs) if delete_tracking_jobs + end + end end def migrate_in(*args, coordinator: coordinator_for_tracking_database) @@ -174,6 +222,10 @@ module Gitlab private + def background_migration_restrict_gitlab_migration_schemas + self.allowed_gitlab_schemas if self.respond_to?(:allowed_gitlab_schemas) + end + def with_migration_context(&block) Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block) end @@ -183,11 +235,9 @@ module Gitlab end def coordinator_for_tracking_database - Gitlab::BackgroundMigration.coordinator_for_database(tracking_database) - end + tracking_database = Gitlab::Database.db_config_name(connection) - def tracking_database - Gitlab::BackgroundMigration::DEFAULT_TRACKING_DATABASE + Gitlab::BackgroundMigration.coordinator_for_database(tracking_database) end end end diff --git a/lib/gitlab/database/migrations/base_background_runner.rb b/lib/gitlab/database/migrations/base_background_runner.rb new file mode 100644 index 00000000000..2772502140e --- /dev/null +++ b/lib/gitlab/database/migrations/base_background_runner.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class BaseBackgroundRunner + attr_reader :result_dir + + def initialize(result_dir:) + @result_dir = result_dir + end + + def jobs_by_migration_name + raise NotImplementedError, 'subclass must implement' + end + + def run_job(job) + raise NotImplementedError, 'subclass must implement' + end + + def run_jobs(for_duration:) + jobs_to_run = jobs_by_migration_name + return if jobs_to_run.empty? + + # without .to_f, we do integer division + # For example, 3.minutes / 2 == 1.minute whereas 3.minutes / 2.to_f == (1.minute + 30.seconds) + duration_per_migration_type = for_duration / jobs_to_run.count.to_f + jobs_to_run.each do |migration_name, jobs| + run_until = duration_per_migration_type.from_now + + run_jobs_for_migration(migration_name: migration_name, jobs: jobs, run_until: run_until) + end + end + + private + + def run_jobs_for_migration(migration_name:, jobs:, run_until:) + per_background_migration_result_dir = File.join(@result_dir, migration_name) + + instrumentation = Instrumentation.new(result_dir: per_background_migration_result_dir) + batch_names = (1..).each.lazy.map { |i| "batch_#{i}"} + + jobs.shuffle.each do |j| + break if run_until <= Time.current + + instrumentation.observe(version: nil, + name: batch_names.next, + connection: ActiveRecord::Migration.connection) do + run_job(j) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 0261ade0fe7..7113c3686f1 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -122,6 +122,22 @@ module Gitlab migration.save! migration end + + def finalize_batched_background_migration(job_class_name:, table_name:, column_name:, job_arguments:) + database_name = Gitlab::Database.db_config_name(connection) + + unless ActiveRecord::Base.configurations.primary?(database_name) + raise 'The `#finalize_background_migration` is currently not supported when running in decomposed database, ' \ + 'and this database is not `main:`. For more information visit: ' \ + 'https://docs.gitlab.com/ee/development/database/migrations_for_multiple_databases.html' + end + + migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration(job_class_name, table_name, column_name, job_arguments) + + raise 'Could not find batched background migration' if migration.nil? + + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize(job_class_name, table_name, column_name, job_arguments, connection: connection) + end end end end diff --git a/lib/gitlab/database/migrations/observers/query_log.rb b/lib/gitlab/database/migrations/observers/query_log.rb index 8ca57bb7df9..543e6b8e302 100644 --- a/lib/gitlab/database/migrations/observers/query_log.rb +++ b/lib/gitlab/database/migrations/observers/query_log.rb @@ -6,7 +6,7 @@ module Gitlab module Observers class QueryLog < MigrationObserver def before - @logger_was = ActiveRecord::Base.logger + @logger_was = ActiveRecord::Base.logger # rubocop:disable Database/MultipleDatabases file_path = File.join(output_dir, "migration.log") @logger = Logger.new(file_path) ActiveRecord::Base.logger = @logger diff --git a/lib/gitlab/database/migrations/observers/query_statistics.rb b/lib/gitlab/database/migrations/observers/query_statistics.rb index 54504646a79..2d026f0c8d2 100644 --- a/lib/gitlab/database/migrations/observers/query_statistics.rb +++ b/lib/gitlab/database/migrations/observers/query_statistics.rb @@ -22,6 +22,7 @@ module Gitlab observation.query_statistics = connection.execute(<<~SQL) SELECT query, calls, total_time, max_time, mean_time, rows FROM pg_stat_statements + WHERE pg_get_userbyid(userid) = current_user ORDER BY total_time DESC SQL end diff --git a/lib/gitlab/database/migrations/reestablished_connection_stack.rb b/lib/gitlab/database/migrations/reestablished_connection_stack.rb new file mode 100644 index 00000000000..d7cf482c32a --- /dev/null +++ b/lib/gitlab/database/migrations/reestablished_connection_stack.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module ReestablishedConnectionStack + # This is workaround for `db:migrate` that switches `ActiveRecord::Base.connection` + # depending on execution. This is subject to be removed once proper fix is implemented: + # https://gitlab.com/gitlab-org/gitlab/-/issues/362341 + # + # In some cases when we run application code we need to restore application connection stack: + # - ApplicationRecord (in fact ActiveRecord::Base): points to main + # - Ci::ApplicationRecord: points to ci + # + # rubocop:disable Database/MultipleDatabases + def with_restored_connection_stack(&block) + original_handler = ActiveRecord::Base.connection_handler + + original_db_config = ActiveRecord::Base.connection_db_config + return yield if ActiveRecord::Base.configurations.primary?(original_db_config.name) + + # If the `ActiveRecord::Base` connection is different than `:main` + # re-establish and configure `SharedModel` context accordingly + # to previously established `ActiveRecord::Base` to allow the application + # code to use `ApplicationRecord` and `Ci::ApplicationRecord` usual way. + # We swap a connection handler as migration context does hold an actual + # connection which we cannot close. + base_model = Gitlab::Database.database_base_models.fetch(original_db_config.name.to_sym) + + # copy connections over to new connection handler + db_configs = original_handler.connection_pool_names.map do |connection_pool_name| + [connection_pool_name.constantize, connection_pool_name.constantize.connection_db_config] + end + + new_handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new + ActiveRecord::Base.connection_handler = new_handler + + db_configs.each do |klass, db_config| + new_handler.establish_connection(db_config, owner_name: klass) + end + + # re-establish ActiveRecord::Base to main + ActiveRecord::Base.establish_connection :main # rubocop:disable Database/EstablishConnection + + Gitlab::Database::SharedModel.using_connection(base_model.connection) do + yield + end + ensure + ActiveRecord::Base.connection_handler = original_handler + new_handler&.clear_all_connections! + end + # rubocop:enable Database/MultipleDatabases + end + end + end +end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index 3b6f52b43a8..4404b5bf961 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -21,6 +21,18 @@ module Gitlab TestBackgroundRunner.new(result_dir: BASE_RESULT_DIR.join('background_migrations')) end + def batched_background_migrations(for_database:) + runner = nil + + # Only one loop iteration since we pass `only:` here + Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + runner = Gitlab::Database::Migrations::TestBatchedBackgroundRunner + .new(result_dir: BASE_RESULT_DIR.join('background_migrations'), connection: connection) + end + + runner + end + def migration_context @migration_context ||= ApplicationRecord.connection.migration_context end diff --git a/lib/gitlab/database/migrations/test_background_runner.rb b/lib/gitlab/database/migrations/test_background_runner.rb index 74e54d62e05..f7713237b38 100644 --- a/lib/gitlab/database/migrations/test_background_runner.rb +++ b/lib/gitlab/database/migrations/test_background_runner.rb @@ -3,11 +3,9 @@ module Gitlab module Database module Migrations - class TestBackgroundRunner - attr_reader :result_dir - + class TestBackgroundRunner < BaseBackgroundRunner def initialize(result_dir:) - @result_dir = result_dir + super(result_dir: result_dir) @job_coordinator = Gitlab::BackgroundMigration.coordinator_for_database(Gitlab::Database::MAIN_DATABASE_NAME) end @@ -15,37 +13,12 @@ module Gitlab @job_coordinator.pending_jobs end - def run_jobs(for_duration:) - jobs_to_run = traditional_background_migrations.group_by { |j| class_name_for_job(j) } - return if jobs_to_run.empty? - - # without .to_f, we do integer division - # For example, 3.minutes / 2 == 1.minute whereas 3.minutes / 2.to_f == (1.minute + 30.seconds) - duration_per_migration_type = for_duration / jobs_to_run.count.to_f - jobs_to_run.each do |migration_name, jobs| - run_until = duration_per_migration_type.from_now - - run_jobs_for_migration(migration_name: migration_name, jobs: jobs, run_until: run_until) - end + def jobs_by_migration_name + traditional_background_migrations.group_by { |j| class_name_for_job(j) } end private - def run_jobs_for_migration(migration_name:, jobs:, run_until:) - per_background_migration_result_dir = File.join(@result_dir, migration_name) - - instrumentation = Instrumentation.new(result_dir: per_background_migration_result_dir) - batch_names = (1..).each.lazy.map { |i| "batch_#{i}"} - - jobs.shuffle.each do |j| - break if run_until <= Time.current - - instrumentation.observe(version: nil, name: batch_names.next, connection: ActiveRecord::Migration.connection) do - run_job(j) - end - end - end - def run_job(job) Gitlab::BackgroundMigration.perform(job.args[0], job.args[1]) end diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb new file mode 100644 index 00000000000..0c6a8d3d856 --- /dev/null +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class TestBatchedBackgroundRunner < BaseBackgroundRunner + attr_reader :connection + + def initialize(result_dir:, connection:) + super(result_dir: result_dir) + @connection = connection + end + + def jobs_by_migration_name + Gitlab::Database::BackgroundMigration::BatchedMigration + .executable + .created_after(2.days.ago) # Simple way to exclude migrations already running before migration testing + .to_h do |migration| + batching_strategy = migration.batch_class.new(connection: connection) + + all_migration_jobs = [] + + min_value = migration.next_min_value + + while (next_bounds = batching_strategy.next_batch( + migration.table_name, + migration.column_name, + batch_min_value: min_value, + batch_size: migration.batch_size, + job_arguments: migration.job_arguments + )) + + batch_min, batch_max = next_bounds + + all_migration_jobs << migration.create_batched_job!(batch_min, batch_max) + min_value = batch_max + 1 + end + + [migration.job_class_name, all_migration_jobs] + end + end + + def run_job(job) + Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new(connection: connection).perform(job) + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 034e18ec9f4..a541ecf5316 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -6,7 +6,6 @@ module Gitlab module TableManagementHelpers include ::Gitlab::Database::SchemaHelpers include ::Gitlab::Database::MigrationHelpers - include ::Gitlab::Database::Migrations::BackgroundMigrationHelpers ALLOWED_TABLES = %w[audit_events web_hook_logs].freeze ERROR_SCOPE = 'table partitioning' diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb index 0c78dda734c..6f64d04270f 100644 --- a/lib/gitlab/database/query_analyzer.rb +++ b/lib/gitlab/database/query_analyzer.rb @@ -30,52 +30,25 @@ module Gitlab end end - def within(user_analyzers = nil) - # Due to singleton nature of analyzers - # only an outer invocation of the `.within` - # is allowed to initialize them - if already_within? - raise 'Query analyzers are already defined, cannot re-define them.' if user_analyzers - - return yield - end - - begin!(user_analyzers || all_analyzers) + def within(analyzers = all_analyzers) + newly_enabled_analyzers = begin!(analyzers) begin yield ensure - end! + end!(newly_enabled_analyzers) end end - def already_within? - # If analyzers are set they are already configured - !enabled_analyzers.nil? - end + # Enable query analyzers (only the ones that were not yet enabled) + # Returns a list of newly enabled analyzers + def begin!(analyzers) + analyzers.select do |analyzer| + next if enabled_analyzers.include?(analyzer) - def process_sql(sql, connection) - analyzers = enabled_analyzers - return unless analyzers&.any? - - parsed = parse(sql, connection) - return unless parsed - - analyzers.each do |analyzer| - next if analyzer.suppressed? && !analyzer.requires_tracking?(parsed) - - analyzer.analyze(parsed) - rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e - # We catch all standard errors to prevent validation errors to introduce fatal errors in production - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) - end - end - - # Enable query analyzers - def begin!(analyzers = all_analyzers) - analyzers = analyzers.select do |analyzer| if analyzer.enabled? analyzer.begin! + enabled_analyzers.append(analyzer) true end @@ -84,25 +57,40 @@ module Gitlab false end - - Thread.current[:query_analyzer_enabled_analyzers] = analyzers end - # Disable enabled query analyzers - def end! - enabled_analyzers.select do |analyzer| + # Disable enabled query analyzers (only the ones that were enabled previously) + def end!(analyzers) + analyzers.each do |analyzer| + next unless enabled_analyzers.delete(analyzer) + analyzer.end! rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end - - Thread.current[:query_analyzer_enabled_analyzers] = nil end private def enabled_analyzers - Thread.current[:query_analyzer_enabled_analyzers] + Thread.current[:query_analyzer_enabled_analyzers] ||= [] + end + + def process_sql(sql, connection) + analyzers = enabled_analyzers + return unless analyzers&.any? + + parsed = parse(sql, connection) + return unless parsed + + analyzers.each do |analyzer| + next if analyzer.suppressed? && !analyzer.requires_tracking?(parsed) + + analyzer.analyze(parsed) + rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e + # We catch all standard errors to prevent validation errors to introduce fatal errors in production + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end end def parse(sql, connection) diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb new file mode 100644 index 00000000000..3de9e8011fb --- /dev/null +++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + # The purpose of this analyzer is to validate if tables observed + # are properly used according to schema used by current connection + class GitlabSchemasValidateConnection < Base + CrossSchemaAccessError = Class.new(QueryAnalyzerError) + + class << self + def enabled? + true + end + + def analyze(parsed) + tables = parsed.pg.select_tables + parsed.pg.dml_tables + table_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables) + return if table_schemas.empty? + + allowed_schemas = ::Gitlab::Database.gitlab_schemas_for_connection(parsed.connection) + return unless allowed_schemas + + invalid_schemas = table_schemas - allowed_schemas + if invalid_schemas.any? + message = "The query tried to access #{tables} (of #{table_schemas.to_a}) " + message += "which is outside of allowed schemas (#{allowed_schemas}) " + message += "for the current connection '#{Gitlab::Database.db_config_name(parsed.connection)}'" + + raise CrossSchemaAccessError, message + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index a53da514df2..e0cb803b872 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -33,7 +33,7 @@ module Gitlab def self.enabled? ::Feature::FlipperFeature.table_exists? && - Feature.enabled?(:detect_cross_database_modification, default_enabled: :yaml) + Feature.enabled?(:detect_cross_database_modification) end def self.requires_tracking?(parsed) diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 91c3fcc7d72..e13dd3b2058 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -16,7 +16,7 @@ module Gitlab REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 def self.enabled? - Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml) + Feature.enabled?(:database_reindexing, type: :ops) end def self.invoke(database = nil) diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb index 563fab692ef..f4c8fca8fa2 100644 --- a/lib/gitlab/database/shared_model.rb +++ b/lib/gitlab/database/shared_model.rb @@ -15,14 +15,16 @@ module Gitlab previous_connection = self.overriding_connection unless previous_connection.nil? || previous_connection.equal?(connection) - raise 'cannot nest connection overrides for shared models with different connections' + raise "Cannot change connection for Gitlab::Database::SharedModel "\ + "from '#{Gitlab::Database.db_config_name(previous_connection)}' "\ + "to '#{Gitlab::Database.db_config_name(connection)}'" end self.overriding_connection = connection yield ensure - self.overriding_connection = nil unless previous_connection.equal?(self.overriding_connection) + self.overriding_connection = previous_connection end def connection diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb index 2d9700cb2bc..1e29ae7761b 100644 --- a/lib/gitlab/database_importers/work_items/base_type_importer.rb +++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb @@ -4,10 +4,18 @@ module Gitlab module DatabaseImporters module WorkItems module BaseTypeImporter - def self.import - ::WorkItems::Type::BASE_TYPES.each do |type, attributes| - ::WorkItems::Type.create!(base_type: type, **attributes.slice(:name, :icon_name)) + def self.upsert_types + current_time = Time.current + + base_types = ::WorkItems::Type::BASE_TYPES.map do |type, attributes| + attributes.slice(:name, :icon_name) + .merge(created_at: current_time, updated_at: current_time, base_type: type) end + + ::WorkItems::Type.upsert_all( + base_types, + unique_by: :idx_work_item_types_on_namespace_id_and_name_null_namespace + ) end end end diff --git a/lib/gitlab/default_branch.rb b/lib/gitlab/default_branch.rb index 6bd9a5675c4..bb540b93a58 100644 --- a/lib/gitlab/default_branch.rb +++ b/lib/gitlab/default_branch.rb @@ -4,7 +4,7 @@ module Gitlab module DefaultBranch def self.value(object: nil) - Feature.enabled?(:main_branch_over_master, object, default_enabled: :yaml) ? 'main' : 'master' + Feature.enabled?(:main_branch_over_master, object) ? 'main' : 'master' end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 61bb0c797b4..d6ee21b93b6 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -50,11 +50,11 @@ module Gitlab end def use_semantic_ipynb_diff? - strong_memoize(:_use_semantic_ipynb_diff) { Feature.enabled?(:ipynb_semantic_diff, repository.project, default_enabled: :yaml) } + strong_memoize(:_use_semantic_ipynb_diff) { Feature.enabled?(:ipynb_semantic_diff, repository.project) } end def use_renderable_diff? - strong_memoize(:_renderable_diff_enabled) { Feature.enabled?(:rendered_diffs_viewer, repository.project, default_enabled: :yaml) } + strong_memoize(:_renderable_diff_enabled) { Feature.enabled?(:rendered_diffs_viewer, repository.project) } end def has_renderable? @@ -386,6 +386,10 @@ module Gitlab strong_memoize(:rendered) { Rendered::Notebook::DiffFile.new(self) } end + def ipynb? + file_path.ends_with?('.ipynb') + end + private def diffable_by_attribute? @@ -415,10 +419,6 @@ module Gitlab new_file? || deleted_file? || content_changed? end - def ipynb? - file_path.ends_with?('.ipynb') - end - # We can't use Object#try because Blob doesn't inherit from Object, but # from BasicObject (via SimpleDelegator). def try_blobs(meth) diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 47f3324752d..225b4f7cf86 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -24,7 +24,7 @@ module Gitlab end def highlight - populate_marker_ranges if Feature.enabled?(:use_marker_ranges, project, default_enabled: :yaml) + populate_marker_ranges if Feature.enabled?(:use_marker_ranges, project) @diff_lines.map.with_index do |diff_line, index| diff_line = diff_line.dup @@ -61,7 +61,7 @@ module Gitlab end def apply_marker_ranges_highlight(diff_line, rich_line, index) - marker_ranges = if Feature.enabled?(:use_marker_ranges, project, default_enabled: :yaml) + marker_ranges = if Feature.enabled?(:use_marker_ranges, project) diff_line.marker_ranges else inline_diffs[index] @@ -83,7 +83,7 @@ module Gitlab return unless diff_file && diff_file.diff_refs return diff_line_highlighting(diff_line, plain: true) if blobs_too_large? - if Feature.enabled?(:diff_line_syntax_highlighting, project, default_enabled: :yaml) + if Feature.enabled?(:diff_line_syntax_highlighting, project) diff_line_highlighting(diff_line) else blob_highlighting(diff_line) diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 12ed11b0140..f950d01fdf0 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -74,8 +74,8 @@ module Gitlab diffable.cache_key, VERSION, diff_options, - Feature.enabled?(:use_marker_ranges, diffable.project, default_enabled: :yaml), - Feature.enabled?(:diff_line_syntax_highlighting, diffable.project, default_enabled: :yaml) + Feature.enabled?(:use_marker_ranges, diffable.project), + Feature.enabled?(:diff_line_syntax_highlighting, diffable.project) ].join(":") end end diff --git a/lib/gitlab/diff/rendered/notebook/diff_file.rb b/lib/gitlab/diff/rendered/notebook/diff_file.rb index cf97569ca31..1f064d8af50 100644 --- a/lib/gitlab/diff/rendered/notebook/diff_file.rb +++ b/lib/gitlab/diff/rendered/notebook/diff_file.rb @@ -13,6 +13,7 @@ module Gitlab LOG_IPYNBDIFF_GENERATED = 'IPYNB_DIFF_GENERATED' LOG_IPYNBDIFF_TIMEOUT = 'IPYNB_DIFF_TIMEOUT' LOG_IPYNBDIFF_INVALID = 'IPYNB_DIFF_INVALID' + LOG_IPYNBDIFF_TRUNCATED = 'IPYNB_DIFF_TRUNCATED' attr_reader :source_diff @@ -60,9 +61,16 @@ module Gitlab def notebook_diff strong_memoize(:notebook_diff) do + if source_diff.old_blob&.truncated? || source_diff.new_blob&.truncated? + log_event(LOG_IPYNBDIFF_TRUNCATED) + next + end + Timeout.timeout(timeout_time) do IpynbDiff.diff(source_diff.old_blob&.data, source_diff.new_blob&.data, - raise_if_invalid_nb: true, diffy_opts: { include_diff_info: true })&.tap do + raise_if_invalid_nb: true, + hide_images: true, + diffy_opts: { include_diff_info: true })&.tap do log_event(LOG_IPYNBDIFF_GENERATED) end end @@ -141,7 +149,7 @@ module Gitlab def log_event(message, error = nil) Gitlab::AppLogger.info({ message: message }) - Gitlab::ErrorTracking.track_exception(error) if error + Gitlab::ErrorTracking.log_exception(error) if error nil end end diff --git a/lib/gitlab/doctor/secrets.rb b/lib/gitlab/doctor/secrets.rb index 44f5c97c70c..cd075569d10 100644 --- a/lib/gitlab/doctor/secrets.rb +++ b/lib/gitlab/doctor/secrets.rb @@ -30,14 +30,35 @@ module Gitlab private + # Skipping initializers may be needed if those attempt to access + # encrypted data on initialization and could fail because of it. + # + # format example: + # { + # model_class => { + # [ + # { action: :create, filters: [:before, :filter_name1] }, + # { action: :update, filters: [:after, :filter_name2] } + # ] + # } + # } + MODEL_INITIALIZERS_TO_SKIP = { + Integration => [ + { action: :initialize, filters: [:after, :initialize_properties] } + ] + }.freeze + def check_model_attributes(models_with_attributes) running_failures = 0 models_with_attributes.each do |model, attributes| failures_per_row = Hash.new { |h, k| h[k] = [] } - model.find_each do |data| - attributes.each do |att| - failures_per_row[data.id] << att unless valid_attribute?(data, att) + + with_skipped_callbacks_for(model) do + model.find_each do |data| + attributes.each do |att| + failures_per_row[data.id] << att unless valid_attribute?(data, att) + end end end @@ -82,6 +103,32 @@ module Gitlab false end + + # WARNING: using this logic in other places than a Rake task will need a + # different approach, as simply setting the callback again is not thread-safe + def with_skipped_callbacks_for(model) + raise StandardError, 'can only be used in a Rake environment' unless Gitlab::Runtime.rake? + + skip_callbacks_for_model(model) + + yield + + skip_callbacks_for_model(model, reset: true) + end + + def skip_callbacks_for_model(model, reset: false) + MODEL_INITIALIZERS_TO_SKIP.each do |klass, initializers| + next unless model <= klass + + initializers.each do |initializer| + if reset + model.set_callback(initializer[:action], *initializer[:filters]) + else + model.skip_callback(initializer[:action], *initializer[:filters]) + end + end + end + end end end end diff --git a/lib/gitlab/email/message/build_ios_app_guide.rb b/lib/gitlab/email/message/build_ios_app_guide.rb new file mode 100644 index 00000000000..4acf558a6a2 --- /dev/null +++ b/lib/gitlab/email/message/build_ios_app_guide.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + class BuildIosAppGuide + include Gitlab::Email::Message::InProductMarketing::Helper + include Gitlab::Routing + + attr_accessor :format + + def initialize(format: :html) + @format = format + end + + def subject_line + s_('InProductMarketing|Get set up to build for iOS') + end + + def title + s_("InProductMarketing|Building for iOS? We've got you covered.") + end + + def body_line1 + s_( + 'InProductMarketing|Want to get your iOS app up and running, including publishing all the way to ' \ + 'TestFlight? Follow our guide to set up GitLab and fastlane to publish iOS apps to the App Store.' + ) + end + + def cta_text + s_('InProductMarketing|Learn how to build for iOS') + end + + def cta_link + action_link(cta_text, 'https://about.gitlab.com/blog/2019/03/06/ios-publishing-with-gitlab-and-fastlane/') + end + + def cta2_text + s_('InProductMarketing|Watch iOS building in action.') + end + + def cta2_link + action_link(cta2_text, 'https://www.youtube.com/watch?v=325FyJt7ZG8') + end + + def logo_path + 'mailers/in_product_marketing/create-0.png' + end + + def unsubscribe + unsubscribe_message + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/base.rb b/lib/gitlab/email/message/in_product_marketing/base.rb index 9b50d86de58..bd20b7e5fc7 100644 --- a/lib/gitlab/email/message/in_product_marketing/base.rb +++ b/lib/gitlab/email/message/in_product_marketing/base.rb @@ -70,14 +70,8 @@ module Gitlab end def unsubscribe - parts = Gitlab.com? ? unsubscribe_com : unsubscribe_self_managed(track, series) - - case format - when :html - parts.join(' ') - else - parts.join("\n" + ' ' * 16) - end + self_managed_preferences_link = marketing_preference_link(track, series) + unsubscribe_message(self_managed_preferences_link) end def progress(current: series + 1, total: total_series, track_name: track.to_s.humanize) @@ -110,26 +104,6 @@ module Gitlab Namespaces::InProductMarketingEmailsService::TRACKS[track][:interval_days].size end - def unsubscribe_com - [ - s_('InProductMarketing|If you no longer wish to receive marketing emails from us,'), - s_('InProductMarketing|you may %{unsubscribe_link} at any time.') % { unsubscribe_link: unsubscribe_link } - ] - end - - def unsubscribe_self_managed(track, series) - [ - s_('InProductMarketing|To opt out of these onboarding emails, %{unsubscribe_link}.') % { unsubscribe_link: unsubscribe_link }, - s_("InProductMarketing|If you don't want to receive marketing emails directly from GitLab, %{marketing_preference_link}.") % { marketing_preference_link: marketing_preference_link(track, series) } - ] - end - - def unsubscribe_link - unsubscribe_url = Gitlab.com? ? '%tag_unsubscribe_url%' : profile_notifications_url - - link(s_('InProductMarketing|unsubscribe'), unsubscribe_url) - end - def marketing_preference_link(track, series) params = { utm_source: 'SM', diff --git a/lib/gitlab/email/message/in_product_marketing/helper.rb b/lib/gitlab/email/message/in_product_marketing/helper.rb index 329cace9e9d..0a0e55c2999 100644 --- a/lib/gitlab/email/message/in_product_marketing/helper.rb +++ b/lib/gitlab/email/message/in_product_marketing/helper.rb @@ -31,8 +31,39 @@ module Gitlab s_('InProductMarketing|%{strong_start}GitLab Inc.%{strong_end} 268 Bush Street, #350, San Francisco, CA 94104, USA').html_safe % strong_options end + def unsubscribe_message(self_managed_preferences_link = nil) + parts = Gitlab.com? ? unsubscribe_com : unsubscribe_self_managed(self_managed_preferences_link) + + case format + when :html + parts.join(' ') + else + parts.join("\n" + ' ' * 16) + end + end + private + def unsubscribe_link + unsubscribe_url = Gitlab.com? ? '%tag_unsubscribe_url%' : profile_notifications_url + + link(s_('InProductMarketing|unsubscribe'), unsubscribe_url) + end + + def unsubscribe_com + [ + s_('InProductMarketing|If you no longer wish to receive marketing emails from us,'), + s_('InProductMarketing|you may %{unsubscribe_link} at any time.') % { unsubscribe_link: unsubscribe_link } + ] + end + + def unsubscribe_self_managed(preferences_link) + [ + s_('InProductMarketing|To opt out of these onboarding emails, %{unsubscribe_link}.') % { unsubscribe_link: unsubscribe_link }, + s_("InProductMarketing|If you don't want to receive marketing emails directly from GitLab, %{marketing_preference_link}.") % { marketing_preference_link: preferences_link } + ] + end + def list(array) case format when :html diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 58e7b2f1b44..4da112bc5a0 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -148,7 +148,7 @@ module Gitlab end def find_first_key_from_received_headers - return unless ::Feature.enabled?(:use_received_header_for_incoming_emails, default_enabled: :yaml) + return unless ::Feature.enabled?(:use_received_header_for_incoming_emails) recipients_from_received_headers.find do |email| key = email_class.key_from_address(email) diff --git a/lib/gitlab/encrypted_ldap_command.rb b/lib/gitlab/encrypted_ldap_command.rb index 3675646185e..5e1eabe7ec6 100644 --- a/lib/gitlab/encrypted_ldap_command.rb +++ b/lib/gitlab/encrypted_ldap_command.rb @@ -15,7 +15,7 @@ module Gitlab <<~YAML # main: # password: '123' - # user_dn: 'gitlab-adm' + # bind_dn: 'gitlab-adm' YAML end end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index d71f9b5e7cf..f9959d5677b 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -116,13 +116,13 @@ module Gitlab private def before_send_raven(event, hint) - return unless Feature.enabled?(:enable_old_sentry_integration, default_enabled: :yaml) + return unless Feature.enabled?(:enable_old_sentry_integration) before_send(event, hint) end def before_send_sentry(event, hint) - return unless Feature.enabled?(:enable_new_sentry_integration, default_enabled: :yaml) + return unless Feature.enabled?(:enable_new_sentry_integration) before_send(event, hint) end diff --git a/lib/gitlab/error_tracking/error_repository.rb b/lib/gitlab/error_tracking/error_repository.rb new file mode 100644 index 00000000000..4ec636703d9 --- /dev/null +++ b/lib/gitlab/error_tracking/error_repository.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + # Data access layer for errors and events related to Error Tracking feature. + class ErrorRepository + Pagination = Struct.new(:next, :prev) + + # Generic database error + DatabaseError = Class.new(StandardError) + # Record was invalid + RecordInvalidError = Class.new(DatabaseError) + + # Builds an instance of error repository backed by a strategy. + # + # @return [self] + def self.build(project) + strategy = ActiveRecordStrategy.new(project) + + new(strategy) + end + + # @private + def initialize(strategy) + @strategy = strategy + end + + # Stores an error and the related error event. + # + # @param name [String] name of the error + # @param description [String] description of the error + # @param actor [String] culprit (class/method/function) which triggered this error + # @param platform [String] platform on which the error occurred + # @param environment [String] environment on which the error occurred + # @param level [String] severity of this error + # @param occurred_at [Time] timestamp when the error occurred + # @param payload [Hash] original error payload + # + # @return [void] nothing + # + # @raise [RecordInvalidError] if passed attributes were invalid to store an error or error event + # @raise [DatabaseError] if generic error occurred + def report_error( + name:, description:, actor:, platform:, + environment:, level:, occurred_at: Time.zone.now, payload: {} + ) + strategy.report_error( + name: name, + description: description, + actor: actor, + platform: platform, + environment: environment, + level: level, + occurred_at: occurred_at, + payload: payload + ) + + nil + end + + # Finds an error by +id+. + # + # @param id [Integer, String] unique error identifier + # + # @return [Gitlab::ErrorTracking::DetailedError] a detail error + def find_error(id) + strategy.find_error(id) + end + + # Lists errors. + # + # @param sort [String] order list by 'first_seen', 'last_seen', or 'frequency' + # @param filters [Hash<Symbol, String>] filter list by + # @option filters [String] :status error status + # @param limit [Integer, String] limit result + # @param cursor [Hash] pagination information + # + # @return [Array<Array<Gitlab::ErrorTracking::Error>, Pagination>] + def list_errors(sort: 'last_seen', filters: {}, limit: 20, cursor: {}) + limit = [limit.to_i, 100].min + + strategy.list_errors(filters: filters, sort: sort, limit: limit, cursor: cursor) + end + + # Fetches last event for error +id+. + # + # @param id [Integer, String] unique error identifier + # + # @return [Gitlab::ErrorTracking::ErrorEvent] + # + # @raise [DatabaseError] if generic error occurred + def last_event_for(id) + strategy.last_event_for(id) + end + + # Updates attributes of an error. + # + # @param id [Integer, String] unique error identifier + # @param status [String] error status + # + # @return [true, false] if update was successful + # + # @raise [DatabaseError] if generic error occurred + def update_error(id, status:) + strategy.update_error(id, status: status) + end + + private + + attr_reader :strategy + end + end +end diff --git a/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb b/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb new file mode 100644 index 00000000000..e5b532ee0f0 --- /dev/null +++ b/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class ErrorRepository + class ActiveRecordStrategy + def initialize(project) + @project = project + end + + def report_error( + name:, description:, actor:, platform:, + environment:, level:, occurred_at:, payload: + ) + error = project_errors.report_error( + name: name, # Example: ActionView::MissingTemplate + description: description, # Example: Missing template posts/show in... + actor: actor, # Example: PostsController#show + platform: platform, # Example: ruby + timestamp: occurred_at + ) + + # The payload field contains all the data on error including stacktrace in jsonb. + # Together with occurred_at these are 2 main attributes that we need to save here. + error.events.create!( + environment: environment, + description: description, + level: level, + occurred_at: occurred_at, + payload: payload + ) + rescue ActiveRecord::ActiveRecordError => e + handle_exceptions(e) + end + + def find_error(id) + project_error(id).to_sentry_detailed_error + rescue ActiveRecord::ActiveRecordError => e + handle_exceptions(e) + end + + def list_errors(filters:, sort:, limit:, cursor:) + errors = project_errors + errors = filter_by_status(errors, filters[:status]) + errors = sort(errors, sort) + errors = errors.keyset_paginate(cursor: cursor, per_page: limit) + + pagination = ErrorRepository::Pagination.new(errors.cursor_for_next_page, errors.cursor_for_previous_page) + + [errors.map(&:to_sentry_error), pagination] + end + + def last_event_for(id) + project_error(id).last_event&.to_sentry_error_event + rescue ActiveRecord::ActiveRecordError => e + handle_exceptions(e) + end + + def update_error(id, **attributes) + project_error(id).update(attributes) + end + + private + + attr_reader :project + + def project_errors + ::ErrorTracking::Error.where(project: project) # rubocop:disable CodeReuse/ActiveRecord + end + + def project_error(id) + project_errors.find(id) + end + + def filter_by_status(errors, status) + return errors unless ::ErrorTracking::Error.statuses.key?(status) + + errors.for_status(status) + end + + def sort(errors, sort) + return errors.order_id_desc unless sort + + errors.sort_by_attribute(sort) + end + + def handle_exceptions(exception) + case exception + when ActiveRecord::RecordInvalid + raise RecordInvalidError, exception.message + else + raise DatabaseError, exception.message + end + end + end + end + end +end diff --git a/lib/gitlab/experiment/rollout/feature.rb b/lib/gitlab/experiment/rollout/feature.rb index 4bef92f5c23..bf31dfe08a0 100644 --- a/lib/gitlab/experiment/rollout/feature.rb +++ b/lib/gitlab/experiment/rollout/feature.rb @@ -14,7 +14,7 @@ module Gitlab def enabled? return false unless feature_flag_defined? return false unless Gitlab.com? - return false unless ::Feature.enabled?(:gitlab_experiment, type: :ops, default_enabled: :yaml) + return false unless ::Feature.enabled?(:gitlab_experiment, type: :ops) feature_flag_instance.state != :off end @@ -29,7 +29,7 @@ module Gitlab # which will assign the control. Otherwise we call super, which will # assign a variant evenly, or based on our provided distribution rules. def execute_assignment - super if ::Feature.enabled?(feature_flag_name, self, type: :experiment, default_enabled: :yaml) + super if ::Feature.enabled?(feature_flag_name, self, type: :experiment) end # This is what's provided to the `Feature.enabled?` call that will be diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index a68e2db4dac..b09d67b8d5f 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -146,9 +146,9 @@ module Gitlab return experimentation_subject_id if subject.blank? if subject.respond_to?(:to_global_id) - Digest::MD5.hexdigest(subject.to_global_id.to_s) + Digest::SHA256.hexdigest(subject.to_global_id.to_s) else - Digest::MD5.hexdigest(subject.to_s) + Digest::SHA256.hexdigest(subject.to_s) end end end diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb index b13f55e7969..0c7091d19e3 100644 --- a/lib/gitlab/experimentation/experiment.rb +++ b/lib/gitlab/experimentation/experiment.rb @@ -16,7 +16,7 @@ module Gitlab def active? # TODO: just touch a feature flag # Temporary change, we will change `experiment_percentage` in future to `Feature.enabled? - Feature.enabled?(feature_flag_name, type: :experiment, default_enabled: :yaml) + Feature.enabled?(feature_flag_name, type: :experiment) ::Gitlab.com? && experiment_percentage > 0 end diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index fbe52db9c0b..9637f8756b1 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -13,10 +13,6 @@ module Gitlab end end - def initialize(repository, name, target, target_commit) - super(repository, name, target, target_commit) - end - def active? self.dereferenced_target.committed_date >= STALE_BRANCH_THRESHOLD.ago end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index a66517b4ca0..c473fe6973d 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -230,12 +230,16 @@ module Gitlab private def encode_diff_to_utf8(replace_invalid_utf8_chars) - return unless Feature.enabled?(:convert_diff_to_utf8_with_replacement_symbol, default_enabled: :yaml) - return unless replace_invalid_utf8_chars && !detect_binary?(@diff) + return unless Feature.enabled?(:convert_diff_to_utf8_with_replacement_symbol) + return unless replace_invalid_utf8_chars && diff_should_be_converted? @diff = Gitlab::EncodingHelper.encode_utf8_with_replacement_character(@diff) end + def diff_should_be_converted? + !detect_binary?(@diff) || !@diff&.valid_encoding? + end + def init_from_hash(hash) raw_diff = hash.symbolize_keys diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index f98fb66ad21..cba63b3c6c7 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -177,8 +177,10 @@ module Gitlab def check_valid_actor! return unless key? - unless actor.valid? + if !actor.valid? raise ForbiddenError, "Your SSH key #{actor.errors[:key].first}." + elsif actor.expired? + raise ForbiddenError, "Your SSH key has expired." end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 4fe5c8df36f..5e1f92ae835 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -276,7 +276,36 @@ module Gitlab ) response = GitalyClient.call(@repository.storage, :commit_service, :list_all_commits, request, timeout: GitalyClient.medium_timeout) - consume_commits_response(response) + + quarantined_commits = consume_commits_response(response) + + if Feature.enabled?(:filter_quarantined_commits) + quarantined_commit_ids = quarantined_commits.map(&:id) + + # While in general the quarantine directory would only contain objects + # which are actually new, this is not guaranteed by Git. In fact, + # git-push(1) may sometimes push objects which already exist in the + # target repository. We do not want to return those from this method + # though given that they're not actually new. + # + # To fix this edge-case we thus have to filter commits down to those + # which don't yet exist. To do so, we must check for object existence + # in the main repository, but the object directory of our repository + # points into the object quarantine. This can be fixed by unsetting + # it, which will cause us to use the normal repository as indicated by + # its relative path again. + main_repo = @gitaly_repo.dup + main_repo.git_object_directory = "" + + # Check object existence of all quarantined commits' IDs. + quarantined_commit_existence = object_existence_map(quarantined_commit_ids, gitaly_repo: main_repo) + + # And then we reject all quarantined commits which exist in the main + # repository already. + quarantined_commits.reject! { |c| quarantined_commit_existence[c.id] } + end + + quarantined_commits else list_commits(Array.wrap(revisions) + %w[--not --all]) end @@ -387,6 +416,35 @@ module Gitlab consume_commits_response(response) end + # Check whether the given revisions exist. Returns a hash mapping the revision name to either `true` if the + # revision exists, or `false` otherwise. This function accepts all revisions as specified by + # gitrevisions(1). + def object_existence_map(revisions, gitaly_repo: @gitaly_repo) + enum = Enumerator.new do |y| + # This is a bug in Gitaly: revisions of the initial request are ignored. This will be fixed in v15.0 via + # https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4510, so we can merge initial request and the initial + # set of revisions starting with v15.1. + y.yield Gitaly::CheckObjectsExistRequest.new(repository: gitaly_repo) + + revisions.each_slice(100) do |revisions_subset| + y.yield Gitaly::CheckObjectsExistRequest.new(revisions: revisions_subset) + end + end + + response = GitalyClient.call( + @repository.storage, :commit_service, :check_objects_exist, enum, timeout: GitalyClient.medium_timeout + ) + + existence_by_revision = {} + response.each do |message| + message.revisions.each do |revision| + existence_by_revision[revision.name] = revision.exists + end + end + + existence_by_revision + end + def filter_shas_with_signatures(shas) request = Gitaly::FilterShasWithSignaturesRequest.new(repository: @gitaly_repo) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 1e199a55b5a..5adb8d946a0 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -48,7 +48,7 @@ module Gitlab def repository_size request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :repository_size, request, timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@storage, :repository_service, :repository_size, request, timeout: GitalyClient.long_timeout) response.size end diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index 7ac0d875512..9556a9e98ba 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -36,7 +36,7 @@ module Gitlab end def self.per_page(project) - if project.group.present? && Feature.enabled?(:github_importer_lower_per_page_limit, project.group, type: :ops, default_enabled: :yaml) + if project.group.present? && Feature.enabled?(:github_importer_lower_per_page_limit, project.group, type: :ops) Gitlab::GithubImport::Client::LOWER_PER_PAGE else Gitlab::GithubImport::Client::DEFAULT_PER_PAGE diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb index 5298a3d81ea..da205ebd345 100644 --- a/lib/gitlab/github_import/issuable_finder.rb +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -78,7 +78,7 @@ module Gitlab end def timeout - if project.group.present? && ::Feature.enabled?(:github_importer_single_endpoint_notes_import, project.group, type: :ops, default_enabled: :yaml) + if project.group.present? && ::Feature.enabled?(:github_importer_single_endpoint_notes_import, project.group, type: :ops) Gitlab::Cache::Import::Caching::LONGER_TIMEOUT else Gitlab::Cache::Import::Caching::TIMEOUT diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 97de2a49e72..ab20b372d53 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -207,13 +207,8 @@ module Gitlab end # Default batch settings for parallel import (can be redefined in Importer classes) - # Example: { size: 100, delay: 1.minute } def parallel_import_batch - if Feature.enabled?(:distribute_github_parallel_import, default_enabled: :yaml) - { size: 1000, delay: 1.minute } - else - {} - end + { size: 1000, delay: 1.minute } end def abort_on_failure diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 3c85d56874f..98570c02e3d 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -51,15 +51,14 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend - push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml) - push_frontend_feature_flag(:security_auto_fix, default_enabled: false) - push_frontend_feature_flag(:new_header_search, default_enabled: :yaml) - push_frontend_feature_flag(:bootstrap_confirmation_modals, default_enabled: :yaml) - push_frontend_feature_flag(:sandboxed_mermaid, default_enabled: :yaml) - push_frontend_feature_flag(:source_editor_toolbar, default_enabled: :yaml) - push_frontend_feature_flag(:gl_avatar_for_all_user_avatars, default_enabled: :yaml) - push_frontend_feature_flag(:mr_attention_requests, default_enabled: :yaml) - push_frontend_feature_flag(:markdown_continue_lists, default_enabled: :yaml) + push_frontend_feature_flag(:usage_data_api, type: :ops) + push_frontend_feature_flag(:security_auto_fix) + push_frontend_feature_flag(:new_header_search) + push_frontend_feature_flag(:bootstrap_confirmation_modals) + push_frontend_feature_flag(:sandboxed_mermaid) + push_frontend_feature_flag(:source_editor_toolbar) + push_frontend_feature_flag(:gl_avatar_for_all_user_avatars) + push_frontend_feature_flag(:mr_attention_requests, current_user) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/find_argument_in_parent.rb b/lib/gitlab/graphql/find_argument_in_parent.rb deleted file mode 100644 index 1f83f8fce7a..00000000000 --- a/lib/gitlab/graphql/find_argument_in_parent.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module FindArgumentInParent - # Searches up the GraphQL AST and returns the first matching argument - # passed to a node - def self.find(parent, argument, limit_depth: nil) - argument = argument.to_s.camelize(:lower).to_sym - depth = 0 - - while parent.respond_to?(:parent) - args = node_args(parent) - return args[argument] if args.key?(argument) - - depth += 1 - return if limit_depth && depth >= limit_depth - - parent = parent.parent - end - end - - class << self - private - - def node_args(node) - node.irep_node.arguments - end - end - end - end -end diff --git a/lib/gitlab/graphql/global_id_compatibility.rb b/lib/gitlab/graphql/global_id_compatibility.rb deleted file mode 100644 index a96e4c4b976..00000000000 --- a/lib/gitlab/graphql/global_id_compatibility.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module GlobalIDCompatibility - # TODO: remove this module once the compatibility layer is no longer needed. - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/257883 - def coerce_global_id_arguments!(args) - global_id_arguments = self.class.arguments.values.select do |arg| - arg.type.is_a?(Class) && arg.type <= ::Types::GlobalIDType - end - - global_id_arguments.each do |arg| - k = arg.keyword - args[k] &&= arg.type.coerce_isolated_input(args[k]) - end - end - end - end -end diff --git a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb index bf9b73d918a..9beb40ddd7e 100644 --- a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb +++ b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb @@ -68,7 +68,7 @@ module Gitlab def items original_items = super - return original_items if Feature.disabled?(:new_graphql_keyset_pagination, default_enabled: :yaml) || Gitlab::Pagination::Keyset::Order.keyset_aware?(original_items) + return original_items if Feature.disabled?(:new_graphql_keyset_pagination) || Gitlab::Pagination::Keyset::Order.keyset_aware?(original_items) strong_memoize(:generic_keyset_pagination_items) do rebuilt_items_with_keyset_order, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(original_items) diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb index 5d3a9245427..cf06a2729d9 100644 --- a/lib/gitlab/graphql/queries.rb +++ b/lib/gitlab/graphql/queries.rb @@ -240,6 +240,9 @@ module Gitlab end end + # TODO: some queries live under app/graphql/queries - we should look there if/when we add fragments there + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/361079 + # for fragments too. class Fragments def initialize(root, dir = 'app/assets/javascripts') @root = root @@ -278,7 +281,7 @@ module Gitlab def self.all ['.', 'ee'].flat_map do |prefix| - find(Rails.root / prefix / 'app/assets/javascripts') + find(Rails.root / prefix / 'app/assets/javascripts') + find(Rails.root / prefix / 'app/graphql/queries') end end diff --git a/lib/gitlab/health_checks/middleware.rb b/lib/gitlab/health_checks/middleware.rb new file mode 100644 index 00000000000..3fe065147c8 --- /dev/null +++ b/lib/gitlab/health_checks/middleware.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + class Middleware + def initialize(app, readiness_probe, liveness_probe) + @app = app + @readiness_probe = readiness_probe + @liveness_probe = liveness_probe + end + + def call(env) + case env['PATH_INFO'] + when '/readiness' then render_probe(@readiness_probe) + when '/liveness' then render_probe(@liveness_probe) + else @app.call(env) + end + end + + private + + def render_probe(probe) + result = probe.execute + + [ + result.http_status, + { 'Content-Type' => 'application/json; charset=utf-8' }, + [result.json.to_json] + ] + end + end + end +end diff --git a/lib/gitlab/health_checks/server.rb b/lib/gitlab/health_checks/server.rb new file mode 100644 index 00000000000..d747b64a221 --- /dev/null +++ b/lib/gitlab/health_checks/server.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'webrick' + +module Gitlab + module HealthChecks + class Server < Daemon + def initialize(address:, port:, **options) + super(**options) + + @address = address + @port = port + end + + private + + def start_working + @server = ::WEBrick::HTTPServer.new( + Port: @port, BindAddress: @address, AccessLog: [] + ) + @server.mount '/', Rack::Handler::WEBrick, rack_app + + true + end + + def run_thread + @server&.start + rescue IOError + # ignore forcibily closed servers + end + + def stop_working + if @server + # we close sockets if thread is not longer running + # this happens, when the process forks + if thread.alive? + @server.shutdown + else + @server.listeners.each(&:close) + end + end + + @server = nil + end + + def rack_app + readiness = new_probe + liveness = new_probe + + Rack::Builder.app do + use Rack::Deflater + use HealthChecks::Middleware, readiness, liveness + run -> (env) { [404, {}, ['']] } + end + end + + def new_probe + ::Gitlab::HealthChecks::Probes::Collection.new + end + end + end +end diff --git a/lib/gitlab/hotlinking_detector.rb b/lib/gitlab/hotlinking_detector.rb index 44901297870..dd58f6aca26 100644 --- a/lib/gitlab/hotlinking_detector.rb +++ b/lib/gitlab/hotlinking_detector.rb @@ -12,7 +12,7 @@ module Gitlab def intercept_hotlinking?(request) request_accepts = parse_request_accepts(request) - return false unless Feature.enabled?(:repository_archive_hotlinking_interception, default_enabled: true) + return false unless Feature.enabled?(:repository_archive_hotlinking_interception) # Block attempts to embed as JS return true if sec_fetch_invalid?(request) diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 8b775d567c8..c8239c9e308 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -43,7 +43,7 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 44, + 'da_DK' => 43, 'de' => 14, 'en' => 100, 'eo' => 0, @@ -53,17 +53,17 @@ module Gitlab 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 34, + 'ja' => 33, 'ko' => 12, 'nb_NO' => 29, 'nl_NL' => 0, 'pl_PL' => 4, 'pt_BR' => 50, - 'ro_RO' => 36, - 'ru' => 31, + 'ro_RO' => 58, + 'ru' => 30, 'tr_TR' => 13, - 'uk' => 46, - 'zh_CN' => 97, + 'uk' => 47, + 'zh_CN' => 96, 'zh_HK' => 2, 'zh_TW' => 2 }.freeze diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 74be56df221..3ad01ef2257 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'securerandom' + module Gitlab module I18n class PoLinter @@ -245,16 +247,24 @@ module Gitlab [] elsif variables.any? { |variable| unnamed_variable?(variable) } variables.map do |variable| - variable == '%d' ? Random.rand(1000) : Gitlab::Utils.random_string + variable == '%d' ? random_number : random_string end else variables.each_with_object({}) do |variable, hash| variable_name = variable[/\w+/] - hash[variable_name] = Gitlab::Utils.random_string + hash[variable_name] = random_string end end end + def random_number + Random.rand(1000) + end + + def random_string + SecureRandom.alphanumeric(64) + end + def validate_unnamed_variables(errors, variables) unnamed_variables, named_variables = variables.partition { |name| unnamed_variable?(name) } diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 829b3771518..1878b5b1a30 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -28,7 +28,7 @@ module Gitlab copy_archive wait_for_archived_file do - validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: :yaml) + validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size) decompress_archive end rescue StandardError => e diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index f7ab1677001..8df5d52bf77 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -16,6 +16,7 @@ tree: - :board - members: - :user + - :namespace_settings included_attributes: user: @@ -24,6 +25,8 @@ included_attributes: - :username author: - :name + namespace_settings: + - :prevent_sharing_groups_outside_hierarchy excluded_attributes: group: @@ -38,6 +41,7 @@ excluded_attributes: - :shared_runners_minute_limit - :extra_shared_runners_minutes_limit - :repository_size_limit + - :max_pages_size epics: - :state_id diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb index adbbd37ce10..258078d595b 100644 --- a/lib/gitlab/import_export/group/relation_factory.rb +++ b/lib/gitlab/import_export/group/relation_factory.rb @@ -30,6 +30,10 @@ module Gitlab update_group_references end + def invalid_relation? + @relation_name == :namespace_settings + end + def update_group_references return unless self.class.existing_object_relations.include?(@relation_name) return unless @relation_hash['group_id'] diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index b44874f598c..4b28dd831fc 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -89,7 +89,7 @@ module Gitlab end def save_relation_object(relation_object, relation_key, relation_definition, relation_index) - if Feature.enabled?(:import_relation_object_persistence, default_enabled: :yaml) && relation_object.new_record? + if Feature.enabled?(:import_relation_object_persistence) && relation_object.new_record? Gitlab::ImportExport::Base::RelationObjectSaver.new( relation_object: relation_object, relation_key: relation_key, diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 7dcf26ca89a..1625c39595c 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -48,6 +48,8 @@ tree: - :award_emoji - releases: - :links + - milestone_releases: + - :milestone - project_members: - :user - merge_requests: @@ -752,6 +754,7 @@ excluded_attributes: - :compliance_framework_setting - :show_default_award_emojis - :warn_about_potentially_unwanted_characters + - :enforce_auth_checks_on_uploads - :services - :exported_protected_branches - :repository_size_limit @@ -957,6 +960,9 @@ excluded_attributes: system_note_metadata: - :description_version_id - :note_id + milestone_releases: + - :milestone_id + - :release_id methods: notes: - :type diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index d8992061524..47f82a901b7 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -54,7 +54,7 @@ module Gitlab end def ndjson_relation_reader - return unless Feature.enabled?(:project_import_ndjson, project.namespace, default_enabled: true) + return unless Feature.enabled?(:project_import_ndjson, project.namespace) ImportExport::Json::NdjsonReader.new( File.join(shared.export_path, 'tree') diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 63c5afa9595..05dcfa5282c 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -80,7 +80,7 @@ module Gitlab def json_writer @json_writer ||= begin - if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace, default_enabled: true) + if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace) full_path = File.join(@shared.export_path, 'tree') Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) else diff --git a/lib/gitlab/inactive_projects_deletion_warning_tracker.rb b/lib/gitlab/inactive_projects_deletion_warning_tracker.rb new file mode 100644 index 00000000000..f3f8e774b4b --- /dev/null +++ b/lib/gitlab/inactive_projects_deletion_warning_tracker.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + class InactiveProjectsDeletionWarningTracker + attr_reader :project_id + + DELETION_TRACKING_REDIS_KEY = 'inactive_projects_deletion_warning_email_notified' + + # Redis key 'inactive_projects_deletion_warning_email_notified' is a hash. It stores the date when the + # deletion warning notification email was sent for an inactive project. The fields and values look like: + # {"project:1"=>"2022-04-22", "project:5"=>"2022-04-22", "project:7"=>"2022-04-25"} + # @return [Hash] + def self.notified_projects + Gitlab::Redis::SharedState.with do |redis| + redis.hgetall(DELETION_TRACKING_REDIS_KEY) + end + end + + def self.reset_all + Gitlab::Redis::SharedState.with do |redis| + redis.del(DELETION_TRACKING_REDIS_KEY) + end + end + + def initialize(project_id) + @project_id = project_id + end + + def notified? + Gitlab::Redis::SharedState.with do |redis| + redis.hexists(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}") + end + end + + def mark_notified + Gitlab::Redis::SharedState.with do |redis| + redis.hset(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}", Date.current) + end + end + + def reset + Gitlab::Redis::SharedState.with do |redis| + redis.hdel(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}") + end + end + end +end diff --git a/lib/gitlab/instrumentation/rate_limiting_gates.rb b/lib/gitlab/instrumentation/rate_limiting_gates.rb new file mode 100644 index 00000000000..960b6995030 --- /dev/null +++ b/lib/gitlab/instrumentation/rate_limiting_gates.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + class RateLimitingGates + GATES = :rate_limiting_gates + + class << self + def track(key) + if ::Gitlab::SafeRequestStore.active? + gates_set << key + end + end + + def gates + gates_set.to_a + end + + def payload + { + GATES => gates + } + end + + private + + def gates_set + ::Gitlab::SafeRequestStore[GATES] ||= Set.new + end + end + end + end +end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 155e365d04c..379c27caeb7 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -32,6 +32,7 @@ module Gitlab instrument_load_balancing(payload) instrument_pid(payload) instrument_uploads(payload) + instrument_rate_limiting_gates(payload) end def instrument_gitaly(payload) @@ -121,6 +122,10 @@ module Gitlab payload.merge! ::Gitlab::Instrumentation::Uploads.payload end + def instrument_rate_limiting_gates(payload) + payload.merge!(::Gitlab::Instrumentation::RateLimitingGates.payload) + end + # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the # `enqueued_at` field or `created_at` field is available. # diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb deleted file mode 100644 index f347db7bc8c..00000000000 --- a/lib/gitlab/integrations/sti_type.rb +++ /dev/null @@ -1,66 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Integrations - class StiType < ActiveRecord::Type::String - NAMESPACED_INTEGRATIONS = %w[ - Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog - Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Harbor Irker Jenkins Jira Mattermost - MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker - Prometheus Pushover Redmine Shimo Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao - ].to_set.freeze - - def self.namespaced_integrations - NAMESPACED_INTEGRATIONS - end - - def cast(value) - new_cast(value) || super - end - - def serialize(value) - new_serialize(value) || super - end - - def deserialize(value) - value - end - - def changed?(original_value, value, _new_value_before_type_cast) - original_value != serialize(value) - end - - def changed_in_place?(original_value_for_database, value) - original_value_for_database != serialize(value) - end - - private - - def new_cast(value) - value = prepare_value(value) - return unless value - - stripped_name = value.delete_suffix('Service') - return unless self.class.namespaced_integrations.include?(stripped_name) - - "Integrations::#{stripped_name}" - end - - def new_serialize(value) - value = prepare_value(value) - return unless value&.starts_with?('Integrations::') - - "#{value.delete_prefix('Integrations::')}Service" - end - - # Returns value cast to a `String`, or `nil` if value is `nil`. - def prepare_value(value) - return value if value.nil? || value.is_a?(String) - - value.to_s - end - end - end -end - -Gitlab::Integrations::StiType.prepend_mod diff --git a/lib/gitlab/project_service_logger.rb b/lib/gitlab/integrations_logger.rb index 9b0357d3161..c62a5f6d321 100644 --- a/lib/gitlab/project_service_logger.rb +++ b/lib/gitlab/integrations_logger.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab - class ProjectServiceLogger < Gitlab::JsonLogger + class IntegrationsLogger < Gitlab::JsonLogger def self.file_name_noext 'integrations_json' end diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index 7abfe8e38e8..02b0c902a70 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -35,6 +35,12 @@ module Gitlab request_params[:base_uri] = uri.to_s request_params.merge!(auth_params) + if Feature.enabled?(:jira_raise_timeouts, type: :ops) + request_params[:open_timeout] = 2.minutes + request_params[:read_timeout] = 2.minutes + request_params[:write_timeout] = 2.minutes + end + result = Gitlab::HTTP.public_send(http_method, path, **request_params) # rubocop:disable GitlabSecurity/PublicSend @authenticated = result.response.is_a?(Net::HTTPOK) store_cookies(result) if options[:use_cookies] diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 9824b46554f..512936bb4f4 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -160,7 +160,7 @@ module Gitlab # @raise [JSON::ParserError] def handle_legacy_mode!(data) return data unless feature_table_exists? - return data unless Feature.enabled?(:json_wrapper_legacy_mode, default_enabled: true) + return data unless Feature.enabled?(:json_wrapper_legacy_mode) raise parser_error if INVALID_LEGACY_TYPES.any? { |type| data.is_a?(type) } end diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index ed7787ffc49..bf7b7f2d089 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -33,6 +33,10 @@ module Gitlab @_version ||= Rails.root.join(VERSION_FILE).read.chomp end + def version_info + Gitlab::VersionInfo.parse(version) + end + # Return GitLab KAS external_url # # @return [String] external_url diff --git a/lib/gitlab/kubernetes/cilium_network_policy.rb b/lib/gitlab/kubernetes/cilium_network_policy.rb deleted file mode 100644 index 8a31e068c30..00000000000 --- a/lib/gitlab/kubernetes/cilium_network_policy.rb +++ /dev/null @@ -1,141 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - class CiliumNetworkPolicy - include NetworkPolicyCommon - extend ::Gitlab::Utils::Override - - API_VERSION = "cilium.io/v2" - KIND = 'CiliumNetworkPolicy' - - PREDEFINED_POLICIES = { - 'allow-inbound-http' => <<~YAML.rstrip, - apiVersion: cilium.io/v2 - kind: CiliumNetworkPolicy - metadata: - name: allow-inbound-http - spec: - endpointSelector: - matchLabels: - network-policy.gitlab.com/disabled_by: gitlab - ingress: - - toPorts: - - ports: - - port: '80' - - port: '443' - YAML - 'drop-outbound' => <<~YAML.rstrip - apiVersion: cilium.io/v2 - kind: CiliumNetworkPolicy - metadata: - name: drop-outbound - spec: - endpointSelector: - matchLabels: - network-policy.gitlab.com/disabled_by: gitlab - egress: - - {} - YAML - }.freeze - - # We are modeling existing kubernetes resource and don't have - # control over amount of parameters. - # rubocop:disable Metrics/ParameterLists - def initialize(name:, namespace:, selector:, ingress:, resource_version: nil, description: nil, labels: nil, creation_timestamp: nil, egress: nil, annotations: nil, environment_ids: []) - @name = name - @description = description - @namespace = namespace - @labels = labels - @creation_timestamp = creation_timestamp - @selector = selector - @resource_version = resource_version - @ingress = ingress - @egress = egress - @annotations = annotations - @environment_ids = environment_ids - end - # rubocop:enable Metrics/ParameterLists - - def self.from_yaml(manifest) - return unless manifest - - policy = YAML.safe_load(manifest, symbolize_names: true) - return if !policy[:metadata] || !policy[:spec] - - metadata = policy[:metadata] - spec = policy[:spec] - self.new( - name: metadata[:name], - description: policy[:description], - namespace: metadata[:namespace], - annotations: metadata[:annotations], - resource_version: metadata[:resourceVersion], - labels: metadata[:labels], - selector: spec[:endpointSelector], - ingress: spec[:ingress], - egress: spec[:egress] - ) - rescue Psych::SyntaxError, Psych::DisallowedClass - nil - end - - def self.from_resource(resource, environment_ids = []) - return unless resource - return if !resource[:metadata] || !resource[:spec] - - metadata = resource[:metadata] - spec = resource[:spec].to_h - self.new( - name: metadata[:name], - description: resource[:description], - namespace: metadata[:namespace], - annotations: metadata[:annotations]&.to_h, - resource_version: metadata[:resourceVersion], - labels: metadata[:labels]&.to_h, - creation_timestamp: metadata[:creationTimestamp], - selector: spec[:endpointSelector], - ingress: spec[:ingress], - egress: spec[:egress], - environment_ids: environment_ids - ) - end - - override :resource - def resource - resource = { - apiVersion: API_VERSION, - kind: KIND, - metadata: metadata, - spec: spec - } - resource[:description] = description if description - resource - end - - private - - attr_reader :name, :description, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress, :annotations, :environment_ids - - def selector - @selector ||= {} - end - - def metadata - meta = { name: name, namespace: namespace } - meta[:labels] = labels if labels - meta[:resourceVersion] = resource_version if resource_version - meta[:annotations] = annotations if annotations - meta - end - - def spec - { - endpointSelector: selector, - ingress: ingress, - egress: egress - }.compact - end - end - end -end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 6caebf445e5..cd03e332175 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -81,24 +81,6 @@ module Gitlab :update_gateway, to: :istio_client - # NetworkPolicy methods delegate to the apis/networking.k8s.io api - # group client - delegate :create_network_policy, - :get_network_policies, - :get_network_policy, - :update_network_policy, - :delete_network_policy, - to: :networking_client - - # CiliumNetworkPolicy methods delegate to the apis/cilium.io api - # group client - delegate :create_cilium_network_policy, - :get_cilium_network_policies, - :get_cilium_network_policy, - :update_cilium_network_policy, - :delete_cilium_network_policy, - to: :cilium_networking_client - attr_reader :api_prefix, :kubeclient_options DEFAULT_KUBECLIENT_OPTIONS = { diff --git a/lib/gitlab/kubernetes/network_policy.rb b/lib/gitlab/kubernetes/network_policy.rb deleted file mode 100644 index e6111db5b17..00000000000 --- a/lib/gitlab/kubernetes/network_policy.rb +++ /dev/null @@ -1,98 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - class NetworkPolicy - include NetworkPolicyCommon - extend ::Gitlab::Utils::Override - - KIND = 'NetworkPolicy' - - # rubocop:disable Metrics/ParameterLists - def initialize(name:, namespace:, selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil, environment_ids: []) - @name = name - @namespace = namespace - @labels = labels - @creation_timestamp = creation_timestamp - @selector = selector - @policy_types = policy_types - @ingress = ingress - @egress = egress - @environment_ids = environment_ids - end - # rubocop:enable Metrics/ParameterLists - - def self.from_yaml(manifest) - return unless manifest - - policy = YAML.safe_load(manifest, symbolize_names: true) - return if !policy[:metadata] || !policy[:spec] - - metadata = policy[:metadata] - spec = policy[:spec] - self.new( - name: metadata[:name], - namespace: metadata[:namespace], - labels: metadata[:labels], - selector: spec[:podSelector], - policy_types: spec[:policyTypes], - ingress: spec[:ingress], - egress: spec[:egress] - ) - rescue Psych::SyntaxError, Psych::DisallowedClass - nil - end - - def self.from_resource(resource, environment_ids = []) - return unless resource - return if !resource[:metadata] || !resource[:spec] - - metadata = resource[:metadata] - spec = resource[:spec].to_h - self.new( - name: metadata[:name], - namespace: metadata[:namespace], - labels: metadata[:labels]&.to_h, - creation_timestamp: metadata[:creationTimestamp], - selector: spec[:podSelector], - policy_types: spec[:policyTypes], - ingress: spec[:ingress], - egress: spec[:egress], - environment_ids: environment_ids - ) - end - - override :resource - def resource - { - kind: KIND, - metadata: metadata, - spec: spec - } - end - - private - - attr_reader :name, :namespace, :labels, :creation_timestamp, :policy_types, :ingress, :egress, :environment_ids - - def selector - @selector ||= {} - end - - def metadata - meta = { name: name, namespace: namespace } - meta[:labels] = labels if labels - meta - end - - def spec - { - podSelector: selector, - policyTypes: policy_types, - ingress: ingress, - egress: egress - } - end - end - end -end diff --git a/lib/gitlab/kubernetes/network_policy_common.rb b/lib/gitlab/kubernetes/network_policy_common.rb deleted file mode 100644 index de91833b734..00000000000 --- a/lib/gitlab/kubernetes/network_policy_common.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module NetworkPolicyCommon - DISABLED_BY_LABEL = :'network-policy.gitlab.com/disabled_by' - - def generate - ::Kubeclient::Resource.new(resource) - end - - def as_json(opts = nil) - { - name: name, - namespace: namespace, - creation_timestamp: creation_timestamp, - manifest: manifest, - is_autodevops: autodevops?, - is_enabled: enabled?, - environment_ids: environment_ids - } - end - - def autodevops? - return false unless labels - - !labels[:chart].nil? && labels[:chart].start_with?('auto-deploy-app-') - end - - # selector selects pods that should be targeted by this - # policy. It can represent podSelector, nodeSelector or - # endpointSelector We can narrow selection by requiring - # this policy to match our custom labels. Since DISABLED_BY - # label will not be on any pod a policy will be effectively disabled. - def enabled? - return true unless selector&.key?(:matchLabels) - - !selector[:matchLabels]&.key?(DISABLED_BY_LABEL) - end - - def enable - return if enabled? - - selector[:matchLabels].delete(DISABLED_BY_LABEL) - end - - def disable - selector[:matchLabels] ||= {} - selector[:matchLabels].merge!(DISABLED_BY_LABEL => 'gitlab') - end - - private - - def resource - raise NotImplementedError - end - - def manifest - YAML.dump(resource.deep_stringify_keys) - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb index 8a176be30a2..e2b43798b22 100644 --- a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb +++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb @@ -33,7 +33,6 @@ module Gitlab def import delete_stale_metrics create_or_update_metrics - update_prometheus_environments end # rubocop: disable CodeReuse/ActiveRecord @@ -47,8 +46,6 @@ module Gitlab affected_metric_ids << prometheus_metric.id end - - @affected_environment_ids += find_alerts(affected_metric_ids).get_environment_id end # rubocop: enable CodeReuse/ActiveRecord @@ -62,24 +59,9 @@ module Gitlab return unless stale_metrics.exists? - delete_stale_alerts(stale_metrics) stale_metrics.each_batch { |batch| batch.delete_all } end - def delete_stale_alerts(stale_metrics) - stale_alerts = find_alerts(stale_metrics) - - affected_environment_ids = stale_alerts.get_environment_id - return unless affected_environment_ids.present? - - @affected_environment_ids += affected_environment_ids - stale_alerts.each_batch { |batch| batch.delete_all } - end - - def find_alerts(metrics) - Projects::Prometheus::AlertsFinder.new(project: project, metric: metrics).execute - end - def prometheus_metrics_attributes @prometheus_metrics_attributes ||= begin Dashboard::Transformers::Yml::V1::PrometheusMetrics.new( @@ -89,19 +71,6 @@ module Gitlab ).execute end end - - def update_prometheus_environments - affected_environments = ::Environment.for_id(@affected_environment_ids.flatten.uniq).for_project(project) - - return unless affected_environments.exists? - - affected_environments.each do |affected_environment| - ::Clusters::Applications::ScheduleUpdateService.new( - affected_environment.cluster_prometheus_adapter, - project - ).execute - end - end end end end diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index 2aea8d655fa..ba2eb729d7b 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -71,28 +71,17 @@ module Gitlab end def rack_app - readiness = readiness_probe - liveness = liveness_probe pid = thread_name gc_requests = @gc_requests Rack::Builder.app do use Rack::Deflater use Gitlab::Metrics::Exporter::MetricsMiddleware, pid - use Gitlab::Metrics::Exporter::HealthChecksMiddleware, readiness, liveness use Gitlab::Metrics::Exporter::GcRequestMiddleware if gc_requests use ::Prometheus::Client::Rack::Exporter if ::Gitlab::Metrics.metrics_folder_present? run -> (env) { [404, {}, ['']] } end end - - def readiness_probe - ::Gitlab::HealthChecks::Probes::Collection.new - end - - def liveness_probe - ::Gitlab::HealthChecks::Probes::Collection.new - end end end end diff --git a/lib/gitlab/metrics/exporter/health_checks_middleware.rb b/lib/gitlab/metrics/exporter/health_checks_middleware.rb deleted file mode 100644 index c43b8004b72..00000000000 --- a/lib/gitlab/metrics/exporter/health_checks_middleware.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Exporter - class HealthChecksMiddleware - def initialize(app, readiness_probe, liveness_probe) - @app = app - @readiness_probe = readiness_probe - @liveness_probe = liveness_probe - end - - def call(env) - case env['PATH_INFO'] - when '/readiness' then render_probe(@readiness_probe) - when '/liveness' then render_probe(@liveness_probe) - else @app.call(env) - end - end - - private - - def render_probe(probe) - result = probe.execute - - [ - result.http_status, - { 'Content-Type' => 'application/json; charset=utf-8' }, - [result.json.to_json] - ] - end - end - end - end -end diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index dc9a7ed1312..0aad865085b 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -56,7 +56,8 @@ module Gitlab end def disabled_by_feature(options) - options.with_feature && !::Feature.enabled?(options.with_feature, type: :ops) + options.with_feature && !::Feature.enabled?(options.with_feature, + type: :undefined, default_enabled_if_undefined: false) end def build_metric!(type, name, options) diff --git a/lib/gitlab/metrics/rails_slis.rb b/lib/gitlab/metrics/rails_slis.rb index c4f305dbdc4..71da0085c8c 100644 --- a/lib/gitlab/metrics/rails_slis.rb +++ b/lib/gitlab/metrics/rails_slis.rb @@ -5,16 +5,16 @@ module Gitlab module RailsSlis class << self def initialize_request_slis! - Gitlab::Metrics::Sli.initialize_sli(:rails_request_apdex, possible_request_labels) unless Gitlab::Metrics::Sli.initialized?(:rails_request_apdex) - Gitlab::Metrics::Sli.initialize_sli(:graphql_query_apdex, possible_graphql_query_labels) unless Gitlab::Metrics::Sli.initialized?(:graphql_query_apdex) + Gitlab::Metrics::Sli::Apdex.initialize_sli(:rails_request, possible_request_labels) + Gitlab::Metrics::Sli::Apdex.initialize_sli(:graphql_query, possible_graphql_query_labels) end def request_apdex - Gitlab::Metrics::Sli[:rails_request_apdex] + Gitlab::Metrics::Sli::Apdex[:rails_request] end def graphql_query_apdex - Gitlab::Metrics::Sli[:graphql_query_apdex] + Gitlab::Metrics::Sli::Apdex[:graphql_query] end private diff --git a/lib/gitlab/metrics/sli.rb b/lib/gitlab/metrics/sli.rb index de73db0755d..fcd893b675f 100644 --- a/lib/gitlab/metrics/sli.rb +++ b/lib/gitlab/metrics/sli.rb @@ -2,12 +2,10 @@ module Gitlab module Metrics - class Sli - SliNotInitializedError = Class.new(StandardError) - + module Sli COUNTER_PREFIX = 'gitlab_sli' - class << self + module ClassMethods INITIALIZATION_MUTEX = Mutex.new def [](name) @@ -16,6 +14,8 @@ module Gitlab def initialize_sli(name, possible_label_combinations) INITIALIZATION_MUTEX.synchronize do + next known_slis[name] if initialized?(name) + sli = new(name) sli.initialize_counters(possible_label_combinations) known_slis[name] = sli @@ -33,6 +33,10 @@ module Gitlab end end + def self.included(mod) + mod.extend(ClassMethods) + end + attr_reader :name def initialize(name) @@ -41,16 +45,17 @@ module Gitlab end def initialize_counters(possible_label_combinations) - @initialized_with_combinations = possible_label_combinations.any? + # This module is effectively an abstract class + @initialized_with_combinations = possible_label_combinations.any? # rubocop:disable Gitlab/ModuleWithInstanceVariables possible_label_combinations.each do |label_combination| total_counter.get(label_combination) - success_counter.get(label_combination) + numerator_counter.get(label_combination) end end - def increment(labels:, success:) + def increment(labels:, increment_numerator:) total_counter.increment(labels) - success_counter.increment(labels) if success + numerator_counter.increment(labels) if increment_numerator end def initialized? @@ -60,23 +65,43 @@ module Gitlab private def total_counter - prometheus.counter(total_counter_name.to_sym, "Total number of measurements for #{name}") + prometheus.counter(counter_name('total'), "Total number of measurements for #{name}") end - def success_counter - prometheus.counter(success_counter_name.to_sym, "Number of successful measurements for #{name}") + def counter_name(suffix) + :"#{COUNTER_PREFIX}:#{name}_#{self.class.name.demodulize.underscore}:#{suffix}" end - def total_counter_name - "#{COUNTER_PREFIX}:#{name}:total" + def prometheus + Gitlab::Metrics end - def success_counter_name - "#{COUNTER_PREFIX}:#{name}:success_total" + class Apdex + include Sli + + def increment(labels:, success:) + super(labels: labels, increment_numerator: success) + end + + private + + def numerator_counter + prometheus.counter(counter_name('success_total'), "Number of successful measurements for #{name}") + end end - def prometheus - Gitlab::Metrics + class ErrorRate + include Sli + + def increment(labels:, error:) + super(labels: labels, increment_numerator: error) + end + + private + + def numerator_counter + prometheus.counter(counter_name('error_total'), "Number of error measurements for #{name}") + end end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 12576cabb19..7c22ce64ea2 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -185,15 +185,17 @@ module Gitlab counters << compose_metric_key(metric, role) end - ::Gitlab::Database.db_config_names.each do |config_name| - counters << compose_metric_key(metric, nil, config_name) # main - counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica + ::Gitlab::Database.database_base_models.keys.each do |config_name| + counters << compose_metric_key(metric, nil, config_name) # main / ci + counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica / ci_replica end end counters end + private_class_method :load_balancing_metric_keys + def compose_metric_key(metric, db_role = nil, db_config_name = nil) self.class.compose_metric_key(metric, db_role, db_config_name) end diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb index 70dcc6fad90..e6cf14a6c8c 100644 --- a/lib/gitlab/metrics/subscribers/rack_attack.rb +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -15,22 +15,6 @@ module Gitlab INSTRUMENTATION_STORE_KEY = :rack_attack_instrumentation - THROTTLES_WITH_USER_INFORMATION = [ - :throttle_authenticated_api, - :throttle_authenticated_web, - :throttle_authenticated_protected_paths_api, - :throttle_authenticated_protected_paths_web, - :throttle_authenticated_packages_api, - :throttle_authenticated_git_lfs, - :throttle_authenticated_files_api, - :throttle_authenticated_deprecated_api - ].freeze - - PAYLOAD_KEYS = [ - :rack_attack_redis_count, - :rack_attack_redis_duration_s - ].freeze - def self.payload Gitlab::SafeRequestStore[INSTRUMENTATION_STORE_KEY] ||= { rack_attack_redis_count: 0, @@ -49,20 +33,20 @@ module Gitlab end def throttle(event) - log_into_auth_logger(event) + log_into_auth_logger(event, status: 429) end def blocklist(event) - log_into_auth_logger(event) + log_into_auth_logger(event, status: 403) end def track(event) - log_into_auth_logger(event) + log_into_auth_logger(event, status: nil) end private - def log_into_auth_logger(event) + def log_into_auth_logger(event, status:) req = event.payload[:request] rack_attack_info = { message: 'Rack_Attack', @@ -73,6 +57,10 @@ module Gitlab matched: req.env['rack.attack.matched'] } + if status + rack_attack_info[:status] = status + end + discriminator = req.env['rack.attack.match_discriminator'].to_s discriminator_id = discriminator.split(':').last diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index bfa4e4cf5f8..dcbb4557377 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -21,6 +21,7 @@ module Gitlab rescue Gitlab::Auth::IpBlacklisted Gitlab::AuthLogger.error( message: 'Rack_Attack', + status: 403, env: :blocklist, remote_ip: request.ip, request_method: request.request_method, diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 51277497c99..b78cd2a6b95 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -136,8 +136,6 @@ module Gitlab def setup_provider(provider) case provider - when :kerberos - require 'omniauth-kerberos' when *omniauth_customized_providers require_dependency "omni_auth/strategies/#{provider}" end diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index e76cab688cc..8bbc9a93610 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -30,11 +30,11 @@ module Gitlab return false unless params[:pagination] == "keyset" if finder.is_a?(BranchesFinder) - Feature.enabled?(:branch_list_keyset_pagination, project, default_enabled: :yaml) + Feature.enabled?(:branch_list_keyset_pagination, project) elsif finder.is_a?(TagsFinder) - Feature.enabled?(:tag_list_keyset_pagination, project, default_enabled: :yaml) + Feature.enabled?(:tag_list_keyset_pagination, project) elsif finder.is_a?(::Repositories::TreeFinder) - Feature.enabled?(:repository_tree_gitaly_pagination, project, default_enabled: :yaml) + Feature.enabled?(:repository_tree_gitaly_pagination, project) else false end @@ -44,11 +44,11 @@ module Gitlab return false unless params[:page].blank? || params[:page].to_i == 1 if finder.is_a?(BranchesFinder) - Feature.enabled?(:branch_list_keyset_pagination, project, default_enabled: :yaml) + Feature.enabled?(:branch_list_keyset_pagination, project) elsif finder.is_a?(TagsFinder) - Feature.enabled?(:tag_list_keyset_pagination, project, default_enabled: :yaml) + Feature.enabled?(:tag_list_keyset_pagination, project) elsif finder.is_a?(::Repositories::TreeFinder) - Feature.enabled?(:repository_tree_gitaly_pagination, project, default_enabled: :yaml) + Feature.enabled?(:repository_tree_gitaly_pagination, project) else false end diff --git a/lib/gitlab/patch/database_config.rb b/lib/gitlab/patch/database_config.rb index 702e8d404b1..c5c73d50518 100644 --- a/lib/gitlab/patch/database_config.rb +++ b/lib/gitlab/patch/database_config.rb @@ -31,10 +31,6 @@ module Gitlab module DatabaseConfig extend ActiveSupport::Concern - prepended do - attr_reader :uses_legacy_database_config - end - def load_database_yaml return super unless Gitlab.ee? @@ -70,24 +66,7 @@ module Gitlab end def database_configuration - @uses_legacy_database_config = false # rubocop:disable Gitlab/ModuleWithInstanceVariables - super.to_h do |env, configs| - # TODO: To be removed in 15.0. See https://gitlab.com/gitlab-org/gitlab/-/issues/338182 - # This preload is needed to convert legacy `database.yml` - # from `production: adapter: postgresql` - # into a `production: main: adapter: postgresql` - unless Gitlab::Utils.to_boolean(ENV['SKIP_DATABASE_CONFIG_VALIDATION'], default: false) - # This check is taken from Rails where the transformation - # of a flat database.yml is done into `primary:` - # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/database_configurations.rb#L169 - if configs.is_a?(Hash) && !configs.all? { |_, v| v.is_a?(Hash) } - configs = { "main" => configs } - - @uses_legacy_database_config = true # rubocop:disable Gitlab/ModuleWithInstanceVariables - end - end - if Gitlab.ee? if !configs.key?("geo") && File.exist?(Rails.root.join("config/database_geo.yml")) configs["geo"] = Rails.application.config_for(:database_geo).stringify_keys diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 6f497c6376d..b05d7160a4b 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -23,7 +23,6 @@ module Gitlab 503.html admin api - apple-touch-icon-precomposed.png apple-touch-icon.png assets dashboard diff --git a/lib/gitlab/phabricator_import.rb b/lib/gitlab/phabricator_import.rb index 4c9d54a93ce..3885a9934d5 100644 --- a/lib/gitlab/phabricator_import.rb +++ b/lib/gitlab/phabricator_import.rb @@ -5,7 +5,7 @@ module Gitlab BaseError = Class.new(StandardError) def self.available? - Feature.enabled?(:phabricator_import, default_enabled: :yaml) && + Feature.enabled?(:phabricator_import) && Gitlab::CurrentSettings.import_sources.include?('phabricator') end end diff --git a/lib/gitlab/process_supervisor.rb b/lib/gitlab/process_supervisor.rb index 18fd24aa582..714034f043d 100644 --- a/lib/gitlab/process_supervisor.rb +++ b/lib/gitlab/process_supervisor.rb @@ -20,7 +20,7 @@ module Gitlab health_check_interval_seconds: DEFAULT_HEALTH_CHECK_INTERVAL_SECONDS, check_terminate_interval_seconds: DEFAULT_TERMINATE_INTERVAL_SECONDS, terminate_timeout_seconds: DEFAULT_TERMINATE_TIMEOUT_SECONDS, - term_signals: %i(INT TERM), + term_signals: [], forwarded_signals: [], **options) super(**options) @@ -31,7 +31,7 @@ module Gitlab @check_terminate_interval_seconds = check_terminate_interval_seconds @terminate_timeout_seconds = terminate_timeout_seconds - @pids = [] + @pids = Set.new @alive = false end @@ -43,7 +43,7 @@ module Gitlab # If the block returns a non-empty list of IDs, the supervisor will # start observing those processes instead. Otherwise it will shut down. def supervise(pid_or_pids, &on_process_death) - @pids = Array(pid_or_pids) + @pids = Array(pid_or_pids).to_set @on_process_death = on_process_death trap_signals! @@ -56,7 +56,6 @@ module Gitlab return unless @alive stop_processes(signal) - stop end def supervised_pids @@ -75,26 +74,25 @@ module Gitlab def run_thread while @alive - sleep(@health_check_interval_seconds) - check_process_health + + sleep(@health_check_interval_seconds) end end def check_process_health unless all_alive? - existing_pids = live_pids # Capture this value for the duration of the block. + existing_pids = live_pids.to_set # Capture this value for the duration of the block. dead_pids = @pids - existing_pids - new_pids = Array(@on_process_death.call(dead_pids)) - @pids = existing_pids + new_pids - @alive = @pids.any? + new_pids = Array(@on_process_death.call(dead_pids.to_a)) + @pids = existing_pids + new_pids.to_set end end def stop_processes(signal) # Set this prior to shutting down so that shutdown hooks which read `alive` # know the supervisor is about to shut down. - @alive = false + stop_working # Shut down supervised processes. signal_all(signal) diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 3a5f1a1d480..d15b57eb888 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -16,7 +16,6 @@ module Gitlab lib/gitlab/middleware/ ee/lib/gitlab/middleware/ lib/gitlab/performance_bar/ - lib/gitlab/request_profiler/ lib/gitlab/query_limiting/ lib/gitlab/tracing/ lib/gitlab/profiler.rb diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 9d954a74948..8a1dcc083e8 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -8,6 +8,7 @@ module Gitlab :assign, :create, :description, + :draft, :label, :merge_when_pipeline_succeeds, :milestone, diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb index 4bfd526914b..49f76ce7814 100644 --- a/lib/gitlab/query_limiting/active_support_subscriber.rb +++ b/lib/gitlab/query_limiting/active_support_subscriber.rb @@ -8,7 +8,7 @@ module Gitlab def sql(event) return if !::Gitlab::QueryLimiting::Transaction.current || event.payload.fetch(:cached, event.payload[:name] == 'CACHE') - ::Gitlab::QueryLimiting::Transaction.current.increment + ::Gitlab::QueryLimiting::Transaction.current.increment(event.payload[:sql]) ::Gitlab::QueryLimiting::Transaction.current.executed_sql(event.payload[:sql]) end end diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index 643b2540c37..2e31849caaa 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -57,12 +57,28 @@ module Gitlab raise(error) if raise_error? end - def increment - @count += 1 if enabled? + def increment(sql = nil) + @count += 1 if enabled? && !ignorable?(sql) + end + + GEO_NODES_LOAD = 'SELECT 1 AS one FROM "geo_nodes" LIMIT 1' + LICENSES_LOAD = 'SELECT "licenses".* FROM "licenses" ORDER BY "licenses"."id"' + ATTR_INTROSPECTION = %r/SELECT .*\ba.attname\b.* (FROM|JOIN) pg_attribute a/m.freeze + + # queries can be safely ignored if they are amoritized in regular usage + # (i.e. only requested occasionally and otherwise cached). + def ignorable?(sql) + return true if sql&.include?(GEO_NODES_LOAD) + return true if sql&.include?(LICENSES_LOAD) + return true if ATTR_INTROSPECTION =~ sql + + false end def executed_sql(sql) - @sql_executed << sql if @count <= LOG_THRESHOLD + return if @count > LOG_THRESHOLD || ignorable?(sql) + + @sql_executed << sql end def raise_error? diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 4efa29337d1..abf55f56c73 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -285,7 +285,7 @@ module Gitlab end types MergeRequest condition do - Feature.enabled?(:mr_attention_requests, project, default_enabled: :yaml) && + current_user.mr_attention_requests_enabled? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end parse_params do |attention_param| @@ -321,7 +321,7 @@ module Gitlab end types MergeRequest condition do - Feature.enabled?(:mr_attention_requests, project, default_enabled: :yaml) && + current_user.mr_attention_requests_enabled? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end parse_params do |attention_param| diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index e4a92ed5122..7ccbeadfd8a 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -10,10 +10,6 @@ module Gitlab @expires_in = expires_in end - def cache_key(key) - super(key) - end - def clear_cache!(key) with do |redis| keys = read(key).map { |value| "#{cache_namespace}:#{value}" } diff --git a/lib/gitlab/repository_archive_rate_limiter.rb b/lib/gitlab/repository_archive_rate_limiter.rb index d395b1aba7f..31a3dc34bf6 100644 --- a/lib/gitlab/repository_archive_rate_limiter.rb +++ b/lib/gitlab/repository_archive_rate_limiter.rb @@ -3,7 +3,7 @@ module Gitlab module RepositoryArchiveRateLimiter def check_archive_rate_limit!(current_user, project, &block) - return unless Feature.enabled?(:archive_rate_limit, default_enabled: :yaml) + return unless Feature.enabled?(:archive_rate_limit) threshold = current_user ? nil : 100 diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb deleted file mode 100644 index 541d505e735..00000000000 --- a/lib/gitlab/request_profiler.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -require 'fileutils' - -module Gitlab - module RequestProfiler - PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles" - - def all - Dir["#{PROFILES_DIR}/*.{html,txt}"].map do |path| - Profile.new(File.basename(path)) - end.select(&:valid?) - end - module_function :all # rubocop: disable Style/AccessModifierDeclarations - - def find(name) - file_path = File.join(PROFILES_DIR, name) - return unless File.exist?(file_path) - - Profile.new(name) - end - module_function :find # rubocop: disable Style/AccessModifierDeclarations - - def profile_token - Rails.cache.fetch('profile-token') do - Devise.friendly_token - end - end - module_function :profile_token # rubocop: disable Style/AccessModifierDeclarations - - def remove_all_profiles - FileUtils.rm_rf(PROFILES_DIR) - end - module_function :remove_all_profiles # rubocop: disable Style/AccessModifierDeclarations - end -end diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb deleted file mode 100644 index acdf8d4541f..00000000000 --- a/lib/gitlab/request_profiler/middleware.rb +++ /dev/null @@ -1,107 +0,0 @@ -# frozen_string_literal: true - -require 'ruby-prof' -require 'memory_profiler' - -module Gitlab - module RequestProfiler - class Middleware - def initialize(app) - @app = app - end - - def call(env) - if profile?(env) - call_with_profiling(env) - else - @app.call(env) - end - end - - def profile?(env) - header_token = env['HTTP_X_PROFILE_TOKEN'] - return unless header_token.present? - - profile_token = Gitlab::RequestProfiler.profile_token - return unless profile_token.present? - - header_token == profile_token - end - - def call_with_profiling(env) - case env['HTTP_X_PROFILE_MODE'] - when 'execution', nil - call_with_call_stack_profiling(env) - when 'memory' - call_with_memory_profiling(env) - else - raise ActionController::BadRequest, invalid_profile_mode(env) - end - end - - def invalid_profile_mode(env) - <<~HEREDOC - Invalid X-Profile-Mode: #{env['HTTP_X_PROFILE_MODE']}. - Supported profile mode request header: - - X-Profile-Mode: execution - - X-Profile-Mode: memory - HEREDOC - end - - def call_with_call_stack_profiling(env) - ret = nil - report = RubyProf::Profile.profile do - ret = catch(:warden) do # rubocop:disable Cop/BanCatchThrow - @app.call(env) - end - end - - generate_report(env, 'execution', 'html') do |file| - printer = RubyProf::CallStackPrinter.new(report) - printer.print(file) - end - - handle_request_ret(ret) - end - - def call_with_memory_profiling(env) - ret = nil - report = MemoryProfiler.report do - ret = catch(:warden) do # rubocop:disable Cop/BanCatchThrow - @app.call(env) - end - end - - generate_report(env, 'memory', 'txt') do |file| - report.pretty_print(to_file: file) - end - - handle_request_ret(ret) - end - - def generate_report(env, report_type, extension) - file_name = "#{env['PATH_INFO'].tr('/', '|')}_#{Time.current.to_i}"\ - "_#{report_type}.#{extension}" - file_path = "#{PROFILES_DIR}/#{file_name}" - - FileUtils.mkdir_p(PROFILES_DIR) - - begin - File.open(file_path, 'wb') do |file| - yield(file) - end - rescue StandardError - FileUtils.rm(file_path) - end - end - - def handle_request_ret(ret) - if ret.is_a?(Array) - ret - else - throw(:warden, ret) # rubocop:disable Cop/BanCatchThrow - end - end - end - end -end diff --git a/lib/gitlab/request_profiler/profile.rb b/lib/gitlab/request_profiler/profile.rb deleted file mode 100644 index 76c675658b1..00000000000 --- a/lib/gitlab/request_profiler/profile.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module RequestProfiler - class Profile - attr_reader :name, :time, :file_path, :request_path, :profile_mode, :type - - alias_method :to_param, :name - - def initialize(name) - @name = name - @file_path = File.join(PROFILES_DIR, name) - - set_attributes - end - - def valid? - @request_path.present? - end - - def content_type - case type - when 'html' - 'text/html' - when 'txt' - 'text/plain' - end - end - - private - - def set_attributes - matches = name.match(/^(?<path>.*)_(?<timestamp>\d+)(_(?<profile_mode>\w+))?\.(?<type>html|txt)$/) - return unless matches - - @request_path = matches[:path].tr('|', '/') - @time = Time.at(matches[:timestamp].to_i).utc - @profile_mode = matches[:profile_mode] || 'unknown' - @type = matches[:type] - end - end - end -end diff --git a/lib/gitlab/safe_request_purger.rb b/lib/gitlab/safe_request_purger.rb new file mode 100644 index 00000000000..b8795f1cc88 --- /dev/null +++ b/lib/gitlab/safe_request_purger.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + class SafeRequestPurger + def self.execute(args) + new(**args).execute + end + + def initialize(resource_key:, resource_ids:) + @resource_key = resource_key + @resource_ids = resource_ids.uniq + @resource_data = {} + end + + def execute + load_resource_data + purge_resource_ids + write_resource_data_to_store + end + + private + + attr_reader :resource_key, :resource_ids, :resource_data + + def load_resource_data + @resource_data = Gitlab::SafeRequestStore.fetch(resource_key) { resource_data } + end + + def purge_resource_ids + @resource_data.delete_if { |id| resource_ids.include?(id) } + end + + def write_resource_data_to_store + Gitlab::SafeRequestStore.write(resource_key, resource_data) + end + end +end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index a498e329c3f..1e42003b203 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -149,28 +149,47 @@ module Gitlab module Praefect extend Gitlab::SetupHelper class << self - def configuration_toml(gitaly_dir, _, _) + def configuration_toml(gitaly_dir, _storage_paths, options) + raise 'This configuration is only intended for test' unless Rails.env.test? + nodes = [{ storage: 'default', address: "unix:#{gitaly_dir}/gitaly.socket", primary: true, token: 'secret' }] second_storage_nodes = [{ storage: 'test_second_storage', address: "unix:#{gitaly_dir}/gitaly2.socket", primary: true, token: 'secret' }] storages = [{ name: 'default', node: nodes }, { name: 'test_second_storage', node: second_storage_nodes }] - failover = { enabled: false, election_strategy: 'local' } + config = { - i_understand_my_election_strategy_is_unsupported_and_will_be_removed_without_warning: true, socket_path: "#{gitaly_dir}/praefect.socket", - memory_queue_enabled: true, virtual_storage: storages, - failover: failover + token: 'secret' } - config[:token] = 'secret' if Rails.env.test? + + if options[:per_repository] + failover = { enabled: true, election_strategy: 'per_repository' } + database = { host: options.fetch(:pghost), + port: options.fetch(:pgport).to_i, + user: options.fetch(:pguser), + dbname: options.fetch(:dbname, 'praefect_test') } + + config.merge!(database: database, + failover: failover) + else + failover = { enabled: false, election_strategy: 'local' } + + config.merge!( + i_understand_my_election_strategy_is_unsupported_and_will_be_removed_without_warning: true, + memory_queue_enabled: true, + failover: failover + ) + end TomlRB.dump(config) end private - def get_config_path(dir, _) - File.join(dir, 'praefect.config.toml') + def get_config_path(dir, options) + config_filename = options[:config_filename] || 'praefect.config.toml' + File.join(dir, config_filename) end end end diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 3eef41a2ca2..ac9a7d25fc2 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -141,6 +141,20 @@ module Gitlab .to_h end + # Get the list of queues from all available workers following queue + # routing rules. Sidekiq::Queue.all fetches the list of queues from Redis. + # It may contain some redundant, obsolete queues from previous iterations + # of GitLab. + def routing_queues + @routing_queues ||= workers.map do |worker| + if worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) + worker.queue + else + ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass) + end + end.uniq.sort + end + private def find_workers(root, ee:, jh:) diff --git a/lib/gitlab/sidekiq_config/dummy_worker.rb b/lib/gitlab/sidekiq_config/dummy_worker.rb index 8a2ea1acaab..ba1f2b8d2ab 100644 --- a/lib/gitlab/sidekiq_config/dummy_worker.rb +++ b/lib/gitlab/sidekiq_config/dummy_worker.rb @@ -22,6 +22,10 @@ module Gitlab @attributes[:queue] end + def queue + @attributes[:queue] + end + def queue_namespace nil end diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb index 1e3fb675ca7..1abdcde6a15 100644 --- a/lib/gitlab/sidekiq_config/worker.rb +++ b/lib/gitlab/sidekiq_config/worker.rb @@ -9,7 +9,7 @@ module Gitlab delegate :feature_category_not_owned?, :generated_queue_name, :get_feature_category, :get_sidekiq_options, :get_tags, :get_urgency, :get_weight, - :get_worker_resource_boundary, :idempotent?, :queue_namespace, + :get_worker_resource_boundary, :idempotent?, :queue_namespace, :queue, :worker_has_external_dependencies?, to: :klass diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index f3e1d0af2aa..dc5481289da 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -43,7 +43,7 @@ module Gitlab metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) - return unless ::Feature.enabled?(:sidekiq_job_completion_metric_initialize, default_enabled: :yaml) + return unless ::Feature.enabled?(:sidekiq_job_completion_metric_initialize) ::Gitlab::SidekiqConfig.current_worker_queue_mappings.each do |worker, queue| worker_class = worker.safe_constantize diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 581d6b738f3..b77f48d1a2c 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -4,10 +4,6 @@ module Gitlab class SnippetSearchResults < SearchResults include SnippetsHelper - def initialize(current_user, query) - super(current_user, query) - end - def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil) paginated_objects(snippet_titles, page, per_page) end diff --git a/lib/gitlab/sourcegraph.rb b/lib/gitlab/sourcegraph.rb index 892c4468107..8369e6fbe9b 100644 --- a/lib/gitlab/sourcegraph.rb +++ b/lib/gitlab/sourcegraph.rb @@ -15,7 +15,7 @@ module Gitlab def feature_enabled?(actor = nil) # Some CI jobs grep for Feature.enabled? in our codebase, so it is important this reference stays around. - Feature.enabled?(:sourcegraph, actor, default_enabled: :yaml) + Feature.enabled?(:sourcegraph, actor) end private diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index b8d124541f9..7ef1be6ff44 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -18,6 +18,10 @@ module Gitlab "#{self.subscriptions_url}/payment_forms/cc_validation" end + def self.payment_validation_form_id + "payment_method_validation" + end + def self.registration_validation_form_url "#{self.subscriptions_url}/payment_forms/cc_registration_validation" end @@ -75,7 +79,7 @@ module Gitlab end def self.renewal_service_email - 'renewals-support@gitlab.com' + 'renewals-service@customers.gitlab.com' end end end @@ -83,5 +87,6 @@ end Gitlab::SubscriptionPortal.prepend_mod Gitlab::SubscriptionPortal::SUBSCRIPTIONS_URL = Gitlab::SubscriptionPortal.subscriptions_url.freeze Gitlab::SubscriptionPortal::PAYMENT_FORM_URL = Gitlab::SubscriptionPortal.payment_form_url.freeze +Gitlab::SubscriptionPortal::PAYMENT_VALIDATION_FORM_ID = Gitlab::SubscriptionPortal.payment_validation_form_id.freeze Gitlab::SubscriptionPortal::RENEWAL_SERVICE_EMAIL = Gitlab::SubscriptionPortal.renewal_service_email.freeze Gitlab::SubscriptionPortal::REGISTRATION_VALIDATION_FORM_URL = Gitlab::SubscriptionPortal.registration_validation_form_url.freeze diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index 323f59d3373..5496bd5f682 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -4,8 +4,7 @@ module Gitlab module Template class GitlabCiYmlTemplate < BaseTemplate BASE_EXCLUDED_PATTERNS = [%r{\.latest\.}].freeze - - TEMPLATES_WITH_LATEST_VERSION = {}.freeze + BASE_DIR = 'lib/gitlab/ci/templates' def description "# This file is a template, and might need editing before it works on your project." @@ -45,7 +44,7 @@ module Gitlab end def base_dir - Rails.root.join('lib/gitlab/ci/templates') + Rails.root.join(BASE_DIR) end def finder(project = nil) @@ -57,31 +56,6 @@ module Gitlab excluded_patterns: self.excluded_patterns ) end - - override :find - def find(key, project = nil) - if try_redirect_to_latest?(key, project) - key += '.latest' - end - - super(key, project) - end - - private - - # To gauge the impact of the latest template, - # you can redirect the stable template to the latest template by enabling the feature flag. - # See https://docs.gitlab.com/ee/development/cicd/templates.html#versioning for more information. - def try_redirect_to_latest?(key, project) - return false unless templates_with_latest_version[key] - - flag_name = "redirect_to_latest_template_#{key.underscore.tr('/', '_')}" - ::Feature.enabled?(flag_name, project, default_enabled: :yaml) - end - - def templates_with_latest_version - TEMPLATES_WITH_LATEST_VERSION - end end end end diff --git a/lib/gitlab/testing/clear_process_memory_cache_middleware.rb b/lib/gitlab/testing/clear_process_memory_cache_middleware.rb index 1e69e5e142d..39bcad271be 100644 --- a/lib/gitlab/testing/clear_process_memory_cache_middleware.rb +++ b/lib/gitlab/testing/clear_process_memory_cache_middleware.rb @@ -11,6 +11,8 @@ module Gitlab Gitlab::ProcessMemoryCache.cache_backend.clear @app.call(env) + ensure + Gitlab::ProcessMemoryCache.cache_backend.clear end end end diff --git a/lib/gitlab/tracking/event_definition.rb b/lib/gitlab/tracking/event_definition.rb index 8f70c8ecab7..928eb6338f6 100644 --- a/lib/gitlab/tracking/event_definition.rb +++ b/lib/gitlab/tracking/event_definition.rb @@ -6,7 +6,6 @@ module Gitlab class EventDefinition EVENT_SCHEMA_PATH = Rails.root.join('config', 'events', 'schema.json') - BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' SCHEMA = ::JSONSchemer.schema(Pathname.new(EVENT_SCHEMA_PATH)) attr_reader :path diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index c0730e7bd59..96e74f00c78 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -61,9 +61,9 @@ module Gitlab def self.with_fallback(pattern, multiline: false) UntrustedRegexp.new(pattern, multiline: multiline) rescue RegexpError - raise if Feature.enabled?(:disable_unsafe_regexp, default_enabled: :yaml) + raise if Feature.enabled?(:disable_unsafe_regexp) - if Feature.enabled?(:ci_unsafe_regexp_logger, type: :ops, default_enabled: :yaml) + if Feature.enabled?(:ci_unsafe_regexp_logger, type: :ops) Gitlab::AppJsonLogger.info( class: self.name, regexp: pattern.to_s, diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index c00a0e1bcb4..a6d6cffec17 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -28,6 +28,8 @@ module Gitlab compare_url(object, **options) when Group instance.group_canonical_url(object, **options) + when WorkItem + instance.work_item_url(object, **options) when Issue instance.issue_url(object, **options) when MergeRequest diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb index 24e044c5740..cf48aa49938 100644 --- a/lib/gitlab/usage/metric.rb +++ b/lib/gitlab/usage/metric.rb @@ -18,19 +18,25 @@ module Gitlab end def with_value - unflatten_key_path(intrumentation_object.value) + with_availability(proc { instrumentation_object.value }) end def with_instrumentation - unflatten_key_path(intrumentation_object.instrumentation) + with_availability(proc { instrumentation_object.instrumentation }) end def with_suggested_name - unflatten_key_path(intrumentation_object.suggested_name) + with_availability(proc { instrumentation_object.suggested_name }) end private + def with_availability(value_proc) + return {} unless instrumentation_object.available? + + unflatten_key_path(value_proc.call) + end + def unflatten_key_path(value) ::Gitlab::Usage::Metrics::KeyPathProcessor.process(definition.key_path, value) end @@ -39,8 +45,8 @@ module Gitlab "Gitlab::Usage::Metrics::Instrumentations::#{definition.instrumentation_class}" end - def intrumentation_object - instrumentation_class.constantize.new( + def instrumentation_object + @instrumentation_object ||= instrumentation_class.constantize.new( time_frame: definition.time_frame, options: definition.attributes[:options] ) diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 1031f38792b..2c50678c6bf 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -4,9 +4,9 @@ module Gitlab module Usage class MetricDefinition METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') - BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze - AVAILABLE_STATUSES = %w[active data_available implemented deprecated].freeze + AVAILABLE_STATUSES = %w[active data_available implemented deprecated].to_set.freeze + VALID_SERVICE_PING_STATUSES = %w[active data_available implemented deprecated broken].to_set.freeze InvalidError = Class.new(RuntimeError) @@ -26,20 +26,22 @@ module Gitlab attributes end + def json_schema + return unless has_json_schema? + + @json_schema ||= Gitlab::Json.parse(File.read(json_schema_path)) + end + def json_schema_path return '' unless has_json_schema? - "#{BASE_REPO_PATH}/#{attributes[:value_json_schema]}" + Rails.root.join(attributes[:value_json_schema]) end def has_json_schema? attributes[:value_type] == 'object' && attributes[:value_json_schema].present? end - def yaml_path - "#{BASE_REPO_PATH}#{path.delete_prefix(Rails.root.to_s)}" - end - def validate! unless skip_validation? self.class.schemer.validate(attributes.stringify_keys).each do |error| @@ -64,6 +66,10 @@ module Gitlab AVAILABLE_STATUSES.include?(attributes[:status]) end + def valid_service_ping_status? + VALID_SERVICE_PING_STATUSES.include?(attributes[:status]) + end + alias_method :to_dictionary, :to_h class << self diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 2545a505984..11e2fd22638 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -30,7 +30,7 @@ module Gitlab def aggregated_metrics_data(start_date:, end_date:, time_frame:) aggregated_metrics.each_with_object({}) do |aggregation, data| - next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: :yaml, type: :development) + next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], type: :development) next unless aggregation[:time_frame].include?(time_frame) case aggregation[:source] diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb index a264f9484f3..f76ed1753b2 100644 --- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb @@ -11,6 +11,18 @@ module Gitlab attr_reader :time_frame attr_reader :options + class << self + def available?(&block) + return @metric_available = block if block_given? + + return @metric_available.call if instance_variable_defined?('@metric_available') + + true + end + + attr_reader :metric_available + end + def initialize(time_frame:, options: {}) @time_frame = time_frame @options = options @@ -19,6 +31,10 @@ module Gitlab def instrumentation value end + + def available? + self.class.available? + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb index 6df6fef5d3a..d42250c9297 100644 --- a/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb @@ -6,7 +6,7 @@ module Gitlab module Instrumentations class CertBasedClustersFfMetric < GenericMetric value do - Feature.enabled?(:certificate_based_clusters, default_enabled: :yaml, type: :ops) + Feature.enabled?(:certificate_based_clusters, type: :ops) end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb index ee51180973c..51be4bf3ccf 100644 --- a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb @@ -6,7 +6,7 @@ module Gitlab module Instrumentations class CollectedDataCategoriesMetric < GenericMetric value do - ::ServicePing::PermitDataCategoriesService.new.execute.to_a + ::ServicePing::PermitDataCategories.new.execute.to_a end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb new file mode 100644 index 00000000000..c0d53b1b21a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountBulkImportsEntitiesMetric < DatabaseMetric + operation :count + + def initialize(time_frame:, options: {}) + super + + if source_type.present? && !source_type.in?(allowed_source_types) + raise ArgumentError, "source_type '#{source_type}' must be one of: #{allowed_source_types.join(', ')}" + end + end + + relation { ::BulkImports::Entity } + + private + + def relation + return super.where(source_type: source_type) if source_type.present? # rubocop: disable CodeReuse/ActiveRecord + + super + end + + def source_type + options[:source_type].to_s + end + + def allowed_source_types + BulkImports::Entity.source_types.keys.map(&:to_s) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb new file mode 100644 index 00000000000..c5498ce530f --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountImportedProjectsMetric < DatabaseMetric + operation :count + + def initialize(time_frame:, options: {}) + super + + raise ArgumentError, "import_type options attribute is required" unless import_type.present? + end + + relation { ::Project } + + start do |time_constraints| + unless time_constraints.nil? + start = time_constraints[:created_at]&.first + + unless start.nil? + ::Project + .select(:id) + .where(Project.arel_table[:created_at].gteq(start)) # rubocop:disable UsageData/LargeTable + .order(created_at: :asc).limit(1).first&.id + end + end + end + + finish do |time_constraints| + unless time_constraints.nil? + finish = time_constraints[:created_at]&.last + + unless finish.nil? + ::Project + .select(:id) + .where(Project.arel_table[:created_at].lteq(finish)) # rubocop:disable UsageData/LargeTable + .order(created_at: :desc).limit(1).first&.id + end + end + end + + private + + def relation + super.imported_from(import_type) # rubocop: disable CodeReuse/ActiveRecord + end + + def import_type + options[:import_type] + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index 34a8bfd08b5..a000b4509c6 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -14,7 +14,14 @@ module Gitlab # ::Issue.where(database_time_constraints) # end # end + + UnimplementedOperationError = Class.new(StandardError) # rubocop:disable UsageData/InstrumentationSuperclass + class << self + IMPLEMENTED_OPERATIONS = %i(count distinct_count estimate_batch_distinct_count).freeze + + private_constant :IMPLEMENTED_OPERATIONS + def start(&block) return @metric_start&.call unless block_given? @@ -40,6 +47,8 @@ module Gitlab end def operation(symbol, column: nil, &block) + raise UnimplementedOperationError unless symbol.in?(IMPLEMENTED_OPERATIONS) + @metric_operation = symbol @column = column @metric_operation_block = block if block_given? @@ -82,6 +91,14 @@ module Gitlab private + def start + self.class.metric_start&.call(time_constraints) + end + + def finish + self.class.metric_finish&.call(time_constraints) + end + def relation self.class.metric_relation.call.where(time_constraints) end @@ -100,19 +117,19 @@ module Gitlab end def get_or_cache_batch_ids - return [self.class.start, self.class.finish] unless self.class.cache_key.present? + return [start, finish] unless self.class.cache_key.present? key_name = "metric_instrumentation/#{self.class.cache_key}" - start = Gitlab::Cache.fetch_once("#{key_name}_minimum_id", expires_in: 1.day) do - self.class.start + cached_start = Gitlab::Cache.fetch_once("#{key_name}_minimum_id", expires_in: 1.day) do + start end - finish = Gitlab::Cache.fetch_once("#{key_name}_maximum_id", expires_in: 1.day) do - self.class.finish + cached_finish = Gitlab::Cache.fetch_once("#{key_name}_maximum_id", expires_in: 1.day) do + finish end - [start, finish] + [cached_start, cached_finish] end end end diff --git a/lib/gitlab/usage/metrics/query.rb b/lib/gitlab/usage/metrics/query.rb index f6947c4c8ff..851aa7a50e8 100644 --- a/lib/gitlab/usage/metrics/query.rb +++ b/lib/gitlab/usage/metrics/query.rb @@ -61,9 +61,31 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def raw_sql(relation, column, distinct = false) column ||= relation.primary_key - relation.select(relation.all.table[column].count(distinct)).to_sql + node = node_to_count(relation, column) + + relation.unscope(:order).select(node.count(distinct)).to_sql + end + # rubocop: enable CodeReuse/ActiveRecord + + def node_to_count(relation, column) + if join_relation?(relation) && joined_column?(column) + table_name, column_name = column.split(".") + Arel::Table.new(table_name)[column_name] + else + relation.all.table[column] + end + end + + def join_relation?(relation) + relation.is_a?(ActiveRecord::Relation) && relation.joins_values.present? + end + + # checks if the passed column is of format "table.column" + def joined_column?(column) + column.is_a?(String) && column.include?(".") end end end diff --git a/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb b/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb new file mode 100644 index 00000000000..e32dcd3777b --- /dev/null +++ b/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module ServicePing + class LegacyMetricTimingDecorator < SimpleDelegator + attr_reader :duration + + delegate :class, :is_a?, :kind_of?, to: :__getobj__ + + def initialize(value, duration) + @duration = duration + super(value) + end + end + end + end +end diff --git a/lib/gitlab/usage/service_ping_report.rb b/lib/gitlab/usage/service_ping_report.rb index 3e653b186a0..e73200cbd4a 100644 --- a/lib/gitlab/usage/service_ping_report.rb +++ b/lib/gitlab/usage/service_ping_report.rb @@ -3,6 +3,8 @@ module Gitlab module Usage class ServicePingReport + CACHE_KEY = 'usage_data' + class << self def for(output:, cached: false) case output.to_sym @@ -26,7 +28,7 @@ module Gitlab end def all_metrics_values(cached) - Rails.cache.fetch('usage_data', force: !cached, expires_in: 2.weeks) do + Rails.cache.fetch(CACHE_KEY, force: !cached, expires_in: 2.weeks) do Gitlab::UsageData.data end end diff --git a/lib/gitlab/usage_counters/common.rb b/lib/gitlab/usage_counters/common.rb deleted file mode 100644 index a5bdac430f4..00000000000 --- a/lib/gitlab/usage_counters/common.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module UsageCounters - class Common - class << self - def increment(project_id) - Gitlab::Redis::SharedState.with { |redis| redis.hincrby(base_key, project_id, 1) } - end - - def usage_totals - Gitlab::Redis::SharedState.with do |redis| - total_sum = 0 - - totals = redis.hgetall(base_key).each_with_object({}) do |(project_id, count), result| - total_sum += result[project_id.to_i] = count.to_i - end - - totals[:total] = total_sum - totals - end - end - - def base_key - raise NotImplementedError - end - end - end - end -end diff --git a/lib/gitlab/usage_counters/pod_logs.rb b/lib/gitlab/usage_counters/pod_logs.rb deleted file mode 100644 index 94e29d2fad7..00000000000 --- a/lib/gitlab/usage_counters/pod_logs.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module UsageCounters - class PodLogs < Common - def self.base_key - 'POD_LOGS_USAGE_COUNTS' - end - end - end -end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index b465d4bcc9b..7a17288e5e5 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -9,7 +9,7 @@ # active_user_count: count(User.active) # alt_usage_data { Gitlab::VERSION } # redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) -# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } +# redis_usage_data { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'users_expanding_vulnerabilities', start_date: 28.days.ago, end_date: Date.current) } # NOTE: # Implementing metrics direct in `usage_data.rb` is deprecated, @@ -308,7 +308,7 @@ module Gitlab Settings[component]['object_store'] end - if config + if config.present? { enabled: alt_usage_data { Settings[component]['enabled'] }, object_store: { @@ -684,6 +684,17 @@ module Gitlab .merge!(ide_monthly_active_users(date_range)) end + def with_duration + return yield unless Feature.enabled?(:measure_service_ping_metric_collection) + + result = nil + duration = Benchmark.realtime do + result = yield + end + + ::Gitlab::Usage::ServicePing::LegacyMetricTimingDecorator.new(result, duration) + end + private def stage_manage_events(time_period) @@ -855,16 +866,17 @@ module Gitlab end def project_imports(time_period) + time_frame = metric_time_period(time_period) counters = { - gitlab_project: projects_imported_count('gitlab_project', time_period), - gitlab: projects_imported_count('gitlab', time_period), - github: projects_imported_count('github', time_period), - bitbucket: projects_imported_count('bitbucket', time_period), - bitbucket_server: projects_imported_count('bitbucket_server', time_period), - gitea: projects_imported_count('gitea', time_period), - git: projects_imported_count('git', time_period), - manifest: projects_imported_count('manifest', time_period), - gitlab_migration: count(::BulkImports::Entity.where(time_period).project_entity) # rubocop: disable CodeReuse/ActiveRecord + gitlab_project: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitlab_project' }), + gitlab: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitlab' }), + github: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'github' }), + bitbucket: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket' }), + bitbucket_server: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket_server' }), + gitea: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitea' }), + git: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'git' }), + manifest: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'manifest' }), + gitlab_migration: add_metric('CountBulkImportsEntitiesMetric', time_frame: time_frame, options: { source_type: :project_entity }) } counters[:total] = add(*counters.values) @@ -872,46 +884,21 @@ module Gitlab counters end - def projects_imported_count(from, time_period) - # rubocop:disable CodeReuse/ActiveRecord - relation = ::Project.imported_from(from).where.not(import_type: nil) # rubocop:disable UsageData/LargeTable - if time_period.empty? - count(relation) - else - @project_import_id ||= {} - start = time_period[:created_at].first - finish = time_period[:created_at].last - - # can be nil values here if no records are in our range and it is possible the same instance - # is called with different time periods since it is passed in as a variable - unless @project_import_id.key?(start) - @project_import_id[start] = ::Project.select(:id).where(Project.arel_table[:created_at].gteq(start)) # rubocop:disable UsageData/LargeTable - .order(created_at: :asc).limit(1).first&.id - end - - unless @project_import_id.key?(finish) - @project_import_id[finish] = ::Project.select(:id).where(Project.arel_table[:created_at].lteq(finish)) # rubocop:disable UsageData/LargeTable - .order(created_at: :desc).limit(1).first&.id - end - - count(relation, start: @project_import_id[start], finish: @project_import_id[finish]) - end - # rubocop:enable CodeReuse/ActiveRecord - end - def issue_imports(time_period) + time_frame = metric_time_period(time_period) { jira: count(::JiraImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord - fogbugz: projects_imported_count('fogbugz', time_period), - phabricator: projects_imported_count('phabricator', time_period), + fogbugz: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'fogbugz' }), + phabricator: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'phabricator' }), csv: count(::Issues::CsvImport.where(time_period)) # rubocop: disable CodeReuse/ActiveRecord } end def group_imports(time_period) + time_frame = metric_time_period(time_period) { group_import: count(::GroupImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord - gitlab_migration: count(::BulkImports::Entity.where(time_period).group_entity) # rubocop: disable CodeReuse/ActiveRecord + gitlab_migration: add_metric('CountBulkImportsEntitiesMetric', time_frame: time_frame, options: { source_type: :group_entity }) } end diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index cf3caf3f0c7..61c071c8738 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -15,7 +15,7 @@ module Gitlab::UsageDataCounters ) namespace = project.namespace - if Feature.enabled?(:route_hll_to_snowplow, namespace, default_enabled: :yaml) + if Feature.enabled?(:route_hll_to_snowplow, namespace) Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, user: user, project: project) end end diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index bc0126cd893..f97ebdccecf 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -8,6 +8,7 @@ module Gitlab EDIT_BY_WEB_IDE = 'g_edit_by_web_ide' EDIT_BY_SSE = 'g_edit_by_sse' EDIT_CATEGORY = 'ide_edit' + EDIT_BY_LIVE_PREVIEW = 'g_edit_by_live_preview' class << self def track_web_ide_edit_action(author:, time: Time.zone.now) @@ -47,6 +48,10 @@ module Gitlab count_unique(EDIT_BY_SSE, date_from, date_to) end + def track_live_preview_edit_action(author:, time: Time.zone.now) + track_unique_action(EDIT_BY_LIVE_PREVIEW, author, time) + end + private def track_unique_action(action, author, time) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 3b34cd77cf5..0ace6e99c59 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -185,7 +185,7 @@ module Gitlab def feature_enabled?(event) return true if event[:feature_flag].blank? - Feature.enabled?(event[:feature_flag], default_enabled: :yaml) && Feature.enabled?(:redis_hll_tracking, type: :ops, default_enabled: :yaml) + Feature.enabled?(event[:feature_flag]) && Feature.enabled?(:redis_hll_tracking, type: :ops) end # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level diff --git a/lib/gitlab/usage_data_counters/ipynb_diff_activity_counter.rb b/lib/gitlab/usage_data_counters/ipynb_diff_activity_counter.rb new file mode 100644 index 00000000000..a34ae909c82 --- /dev/null +++ b/lib/gitlab/usage_data_counters/ipynb_diff_activity_counter.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# noinspection RubyConstantNamingConvention +module Gitlab + module UsageDataCounters + module IpynbDiffActivityCounter + NOTE_CREATED_IN_IPYNB_DIFF_ACTION = 'i_code_review_create_note_in_ipynb_diff' + USER_CREATED_NOTE_IN_IPYNB_DIFF_ACTION = 'i_code_review_user_create_note_in_ipynb_diff' + NOTE_CREATED_IN_IPYNB_DIFF_MR_ACTION = 'i_code_review_create_note_in_ipynb_diff_mr' + USER_CREATED_NOTE_IN_IPYNB_DIFF_MR_ACTION = 'i_code_review_user_create_note_in_ipynb_diff_mr' + NOTE_CREATED_IN_IPYNB_DIFF_COMMIT_ACTION = 'i_code_review_create_note_in_ipynb_diff_commit' + USER_CREATED_NOTE_IN_IPYNB_DIFF_COMMIT_ACTION = 'i_code_review_user_create_note_in_ipynb_diff_commit' + + class << self + def note_created(note) + return unless note.for_merge_request? || note.for_commit? + + if note.for_merge_request? + track(NOTE_CREATED_IN_IPYNB_DIFF_MR_ACTION, USER_CREATED_NOTE_IN_IPYNB_DIFF_MR_ACTION, note) + else + track(NOTE_CREATED_IN_IPYNB_DIFF_COMMIT_ACTION, USER_CREATED_NOTE_IN_IPYNB_DIFF_COMMIT_ACTION, note) + end + + track(NOTE_CREATED_IN_IPYNB_DIFF_ACTION, USER_CREATED_NOTE_IN_IPYNB_DIFF_ACTION, note) + end + + private + + def track(action, per_user_action, note) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, note.id) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(per_user_action, note.author_id) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index f179f6d679d..3b883e505f8 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -127,6 +127,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_security_sast_iac + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_security_dependency_scanning category: ci_templates redis_slot: ci_templates @@ -147,10 +151,6 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_security_cluster_image_scanning - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - name: p_ci_templates_qualys_iac_security category: ci_templates redis_slot: ci_templates @@ -187,6 +187,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_liquibase + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_flutter category: ci_templates redis_slot: ci_templates @@ -207,10 +211,6 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_managed_cluster_applications - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - name: p_ci_templates_php category: ci_templates redis_slot: ci_templates @@ -231,10 +231,6 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_serverless - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - name: p_ci_templates_go category: ci_templates redis_slot: ci_templates @@ -255,6 +251,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_matlab + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_deploy_ecs category: ci_templates redis_slot: ci_templates @@ -331,6 +331,18 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_jobs_sast_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_sast_iac + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_secret_detection_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_jobs_dependency_scanning category: ci_templates redis_slot: ci_templates @@ -523,6 +535,18 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_jobs_sast_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_sast_iac + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_secret_detection_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_jobs_dependency_scanning category: ci_templates redis_slot: ci_templates @@ -595,6 +619,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_security_sast_iac + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_security_dependency_scanning category: ci_templates redis_slot: ci_templates @@ -615,15 +643,3 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_cluster_image_scanning - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_liquibase - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_matlab - category: ci_templates - redis_slot: ci_templates - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_users.yml b/lib/gitlab/usage_data_counters/known_events/ci_users.yml index 63498a35858..5159dcf62ab 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_users.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_users.yml @@ -2,4 +2,4 @@ category: ci_users redis_slot: ci_users aggregation: weekly - feature_flag: job_deployment_count + feature_flag: diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index df2864bba89..e3bb3f6fef3 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -173,6 +173,30 @@ redis_slot: code_review category: code_review aggregation: weekly +- name: i_code_review_create_note_in_ipynb_diff + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_create_note_in_ipynb_diff + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_create_note_in_ipynb_diff_mr + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_create_note_in_ipynb_diff_mr + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_create_note_in_ipynb_diff_commit + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_create_note_in_ipynb_diff_commit + redis_slot: code_review + category: code_review + aggregation: weekly # Diff settings events - name: i_code_review_click_diff_view_setting redis_slot: code_review @@ -234,6 +258,7 @@ redis_slot: code_review category: code_review aggregation: weekly + feature_flag: usage_data_diff_searches - name: i_code_review_total_suggestions_applied redis_slot: code_review category: code_review diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 0d89a5181ec..448ed4c66e1 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -40,6 +40,11 @@ redis_slot: edit expiry: 29 aggregation: daily +- name: g_edit_by_live_preview + category: ide_edit + redis_slot: edit + expiry: 29 + aggregation: daily - name: i_search_total category: search redis_slot: search diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml index 82787b7bf29..dd6625a9cc9 100644 --- a/lib/gitlab/usage_data_counters/known_events/epic_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/epic_events.yml @@ -218,3 +218,10 @@ redis_slot: project_management aggregation: daily feature_flag: track_epics_activity + +- name: g_project_management_epic_blocked_removed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb index 1661a1b6987..79d4b45a1ce 100644 --- a/lib/gitlab/usage_data_non_sql_metrics.rb +++ b/lib/gitlab/usage_data_non_sql_metrics.rb @@ -31,6 +31,10 @@ module Gitlab SQL_METRIC_DEFAULT end + def add(*args) + SQL_METRIC_DEFAULT + end + def maximum_id(model, column = nil) end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index 977cc3549d8..b2d74b1f0dd 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -5,6 +5,10 @@ module Gitlab # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41091 class UsageDataQueries < UsageData class << self + def with_duration + yield + end + def add_metric(metric, time_frame: 'none', options: {}) metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index a4a1cccf9d5..c2f61741cc5 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -103,9 +103,7 @@ module Gitlab def branch_allows_collaboration_for?(ref) return false if skip_collaboration_check - # Checking for an internal project or group to prevent an infinite loop: - # https://gitlab.com/gitlab-org/gitlab/issues/36805 - (!project.internal? && project.branch_allows_collaboration?(user, ref)) + project.branch_allows_collaboration?(user, ref) end def permission_cache diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 816ede4136a..a67a0758257 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -128,10 +128,6 @@ module Gitlab end end - def random_string - Random.rand(Float::MAX.to_i).to_s(36) - end - # Behaves like `which` on Linux machines: given PATH, try to resolve the given # executable name to an absolute path, or return nil. # diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 6c182f98dd0..633f4683b6b 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -31,7 +31,7 @@ # # Examples: # redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) -# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } +# redis_usage_data { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'users_expanding_vulnerabilities', start_date: 28.days.ago, end_date: Date.current) } module Gitlab module Utils @@ -44,57 +44,64 @@ module Gitlab DISTRIBUTED_HLL_FALLBACK = -2 MAX_BUCKET_SIZE = 100 + def with_duration + yield + end + def add_metric(metric, time_frame: 'none', options: {}) metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize metric_class.new(time_frame: time_frame, options: options).value end - def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - if batch - Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish) - else - relation.count + def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil, start_at: Time.current) + with_duration do + if batch + Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish) + else + relation.count + end + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + FALLBACK end - rescue ActiveRecord::StatementInvalid => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - FALLBACK end def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - if batch - Gitlab::Database::BatchCount.batch_distinct_count(relation, column, batch_size: batch_size, start: start, finish: finish) - else - relation.distinct_count_by(column) + with_duration do + if batch + Gitlab::Database::BatchCount.batch_distinct_count(relation, column, batch_size: batch_size, start: start, finish: finish) + else + relation.distinct_count_by(column) + end + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + FALLBACK end - rescue ActiveRecord::StatementInvalid => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - FALLBACK end def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) - buckets = Gitlab::Database::PostgresHll::BatchDistinctCounter - .new(relation, column) - .execute(batch_size: batch_size, start: start, finish: finish) + with_duration do + buckets = Gitlab::Database::PostgresHll::BatchDistinctCounter + .new(relation, column) + .execute(batch_size: batch_size, start: start, finish: finish) - yield buckets if block_given? + yield buckets if block_given? - buckets.estimated_distinct_count - rescue ActiveRecord::StatementInvalid => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - FALLBACK - # catch all rescue should be removed as a part of feature flag rollout issue - # https://gitlab.com/gitlab-org/gitlab/-/issues/285485 - rescue StandardError => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - DISTRIBUTED_HLL_FALLBACK + buckets.estimated_distinct_count + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + FALLBACK + end end def sum(relation, column, batch_size: nil, start: nil, finish: nil) - Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish) - rescue ActiveRecord::StatementInvalid => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - FALLBACK + with_duration do + Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish) + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + FALLBACK + end end # We don't support batching with histograms. @@ -103,103 +110,113 @@ module Gitlab # # rubocop: disable CodeReuse/ActiveRecord def histogram(relation, column, buckets:, bucket_size: buckets.size) - # Using lambda to avoid exposing histogram specific methods - parameters_valid = lambda do - error_message = - if buckets.first == buckets.last - 'Lower bucket bound cannot equal to upper bucket bound' - elsif bucket_size == 0 - 'Bucket size cannot be zero' - elsif bucket_size > MAX_BUCKET_SIZE - "Bucket size #{bucket_size} exceeds the limit of #{MAX_BUCKET_SIZE}" - end - - return true unless error_message - - exception = ArgumentError.new(error_message) - exception.set_backtrace(caller) - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(exception) + with_duration do + # Using lambda to avoid exposing histogram specific methods + parameters_valid = lambda do + error_message = + if buckets.first == buckets.last + 'Lower bucket bound cannot equal to upper bucket bound' + elsif bucket_size == 0 + 'Bucket size cannot be zero' + elsif bucket_size > MAX_BUCKET_SIZE + "Bucket size #{bucket_size} exceeds the limit of #{MAX_BUCKET_SIZE}" + end + + break true unless error_message + + exception = ArgumentError.new(error_message) + exception.set_backtrace(caller) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(exception) + + false + end - false + break HISTOGRAM_FALLBACK unless parameters_valid.call + + count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) + cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) + + # For example, 9 segments gives 10 buckets + bucket_segments = bucket_size - 1 + + width_bucket = Arel::Nodes::NamedFunction + .new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments]) + .as('buckets') + + query = cte + .table + .project(width_bucket, cte.table[:count]) + .group('buckets') + .order('buckets') + .with(cte.to_arel) + + # Return the histogram as a Hash because buckets are unique. + relation + .connection + .exec_query(query.to_sql) + .rows + .to_h + # Keys are converted to strings in Usage Ping JSON + .stringify_keys + rescue ActiveRecord::StatementInvalid => e + Gitlab::AppJsonLogger.error( + event: 'histogram', + relation: relation.table_name, + operation: 'histogram', + operation_args: [column, buckets.first, buckets.last, bucket_segments], + query: query.to_sql, + message: e.message + ) + # Raises error for dev env + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + HISTOGRAM_FALLBACK end - - return HISTOGRAM_FALLBACK unless parameters_valid.call - - count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) - cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) - - # For example, 9 segments gives 10 buckets - bucket_segments = bucket_size - 1 - - width_bucket = Arel::Nodes::NamedFunction - .new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments]) - .as('buckets') - - query = cte - .table - .project(width_bucket, cte.table[:count]) - .group('buckets') - .order('buckets') - .with(cte.to_arel) - - # Return the histogram as a Hash because buckets are unique. - relation - .connection - .exec_query(query.to_sql) - .rows - .to_h - # Keys are converted to strings in Usage Ping JSON - .stringify_keys - rescue ActiveRecord::StatementInvalid => e - Gitlab::AppJsonLogger.error( - event: 'histogram', - relation: relation.table_name, - operation: 'histogram', - operation_args: [column, buckets.first, buckets.last, bucket_segments], - query: query.to_sql, - message: e.message - ) - # Raises error for dev env - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) - HISTOGRAM_FALLBACK end # rubocop: enable CodeReuse/ActiveRecord def add(*args) - return -1 if args.any?(&:negative?) + with_duration do + break -1 if args.any?(&:negative?) - args.sum - rescue StandardError => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - FALLBACK + args.sum + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + FALLBACK + end end def alt_usage_data(value = nil, fallback: FALLBACK, &block) - if block_given? - yield - else - value + with_duration do + if block_given? + yield + else + value + end + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + fallback end - rescue StandardError => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - fallback end def redis_usage_data(counter = nil, &block) - if block_given? - redis_usage_counter(&block) - elsif counter.present? - redis_usage_data_totals(counter) + with_duration do + if block_given? + redis_usage_counter(&block) + elsif counter.present? + redis_usage_data_totals(counter) + end end end def with_prometheus_client(fallback: {}, verify: true) - client = prometheus_client(verify: verify) - return fallback unless client + with_duration do + client = prometheus_client(verify: verify) + break fallback unless client - yield client - rescue StandardError - fallback + yield client + rescue StandardError + fallback + end end def measure_duration @@ -231,25 +248,28 @@ module Gitlab # rubocop: disable UsageData/LargeTable: def jira_integration_data - data = { - projects_jira_server_active: 0, - projects_jira_cloud_active: 0 - } - - # rubocop: disable CodeReuse/ActiveRecord - ::Integrations::Jira.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| - counts = services.group_by do |service| - # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 - service_url = service.data_fields&.url || (service.properties && service.properties['url']) - service_url&.include?('.atlassian.net') ? :cloud : :server + with_duration do + data = { + projects_jira_server_active: 0, + projects_jira_cloud_active: 0 + } + + # rubocop: disable CodeReuse/ActiveRecord + ::Integrations::Jira.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| + counts = services.group_by do |service| + # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 + service_url = service.data_fields&.url || (service.properties && service.properties['url']) + service_url&.include?('.atlassian.net') ? :cloud : :server + end + + data[:projects_jira_server_active] += counts[:server].size if counts[:server] + data[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud] end - data[:projects_jira_server_active] += counts[:server].size if counts[:server] - data[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud] + data end - - data end + # rubocop: enable CodeReuse/ActiveRecord # rubocop: enable UsageData/LargeTable: @@ -263,9 +283,11 @@ module Gitlab end def epics_deepest_relationship_level - # rubocop: disable UsageData/LargeTable - { epics_deepest_relationship_level: ::Epic.deepest_relationship_level.to_i } - # rubocop: enable UsageData/LargeTable + with_duration do + # rubocop: disable UsageData/LargeTable + { epics_deepest_relationship_level: ::Epic.deepest_relationship_level.to_i } + # rubocop: enable UsageData/LargeTable + end end private diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 19d30daa577..e81670ce89a 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -33,8 +33,7 @@ module Gitlab GitalyServer: { address: Gitlab::GitalyClient.address(repository.storage), token: Gitlab::GitalyClient.token(repository.storage), - features: Feature::Gitaly.server_feature_flags(repository.project), - sidechannel: Feature.enabled?(:workhorse_use_sidechannel, repository.project, default_enabled: :yaml) + features: Feature::Gitaly.server_feature_flags(repository.project) } } @@ -226,6 +225,13 @@ module Gitlab end end + def detect_content_type + [ + Gitlab::Workhorse::DETECT_HEADER, + 'true' + ] + end + protected # This is the outermost encoding of a senddata: header. It is safe for diff --git a/lib/gitlab/zentao/client.rb b/lib/gitlab/zentao/client.rb index 8acfb4913f3..4da4631eecf 100644 --- a/lib/gitlab/zentao/client.rb +++ b/lib/gitlab/zentao/client.rb @@ -58,7 +58,7 @@ module Gitlab def url(path) host = integration.api_url.presence || integration.url - URI.join(host, '/api.php/v1/', path) + URI.parse(Gitlab::Utils.append_path(host, "api.php/v1/#{path}")) end def headers |