diff options
Diffstat (limited to 'lib/gitlab')
179 files changed, 9612 insertions, 885 deletions
diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index d93067c7e2f..b10330914ca 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -16,6 +16,8 @@ module Gitlab :client_id, :caller_id, :remote_ip, + :job_id, + :pipeline_id, :related_class, :feature_category ].freeze @@ -28,6 +30,7 @@ module Gitlab Attribute.new(:runner, ::Ci::Runner), Attribute.new(:caller_id, String), Attribute.new(:remote_ip, String), + Attribute.new(:job, ::Ci::Build), Attribute.new(:related_class, String), Attribute.new(:feature_category, String) ].freeze @@ -73,14 +76,16 @@ module Gitlab def to_lazy_hash {}.tap do |hash| - hash[:user] = -> { username } if set_values.include?(:user) - hash[:project] = -> { project_path } if set_values.include?(:project) || set_values.include?(:runner) + hash[:user] = -> { username } if include_user? + hash[:project] = -> { project_path } if include_project? hash[:root_namespace] = -> { root_namespace_path } if include_namespace? hash[:client_id] = -> { client } if include_client? hash[:caller_id] = caller_id if set_values.include?(:caller_id) hash[:remote_ip] = remote_ip if set_values.include?(:remote_ip) hash[:related_class] = related_class if set_values.include?(:related_class) hash[:feature_category] = feature_category if set_values.include?(:feature_category) + hash[:pipeline_id] = -> { job&.pipeline_id } if set_values.include?(:job) + hash[:job_id] = -> { job&.id } if set_values.include?(:job) end end @@ -103,32 +108,41 @@ module Gitlab end def project_path - associated_routable = project || runner_project + associated_routable = project || runner_project || job_project associated_routable&.full_path end def username - user&.username + associated_user = user || job_user + associated_user&.username end def root_namespace_path - associated_routable = namespace || project || runner_project || runner_group + associated_routable = namespace || project || runner_project || runner_group || job_project associated_routable&.full_path_components&.first end def include_namespace? - set_values.include?(:namespace) || set_values.include?(:project) || set_values.include?(:runner) + set_values.include?(:namespace) || set_values.include?(:project) || set_values.include?(:runner) || set_values.include?(:job) end def include_client? set_values.include?(:user) || set_values.include?(:runner) || set_values.include?(:remote_ip) end + def include_user? + set_values.include?(:user) || set_values.include?(:job) + end + + def include_project? + set_values.include?(:project) || set_values.include?(:runner) || set_values.include?(:job) + end + def client - if user - "user/#{user.id}" - elsif runner + if runner "runner/#{runner.id}" + elsif user + "user/#{user.id}" else "ip/#{remote_ip}" end @@ -150,6 +164,18 @@ module Gitlab runner.groups.first end end + + def job_project + strong_memoize(:job_project) do + job&.project + end + end + + def job_user + strong_memoize(:job_user) do + job&.user + end + end end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 0b0aaacbaff..09775297def 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -41,7 +41,8 @@ module Gitlab auto_rollback_deployment: { threshold: 1, interval: 3.minutes }, search_rate_limit: { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute }, search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, - gitlab_shell_operation: { threshold: 600, interval: 1.minute } + gitlab_shell_operation: { threshold: 600, interval: 1.minute }, + pipelines_create: { threshold: 25, interval: 1.minute } }.freeze end diff --git a/lib/gitlab/asciidoc/include_processor.rb b/lib/gitlab/asciidoc/include_processor.rb index 53d1135a2d7..6c4ecc04cdc 100644 --- a/lib/gitlab/asciidoc/include_processor.rb +++ b/lib/gitlab/asciidoc/include_processor.rb @@ -33,7 +33,7 @@ module Gitlab max_include_depth = doc.attributes.fetch('max-include-depth').to_i return false if max_include_depth < 1 - return false if target_uri?(target) + return false if target_http?(target) return false if included.size >= max_includes true diff --git a/lib/gitlab/auth/ldap/dn.rb b/lib/gitlab/auth/ldap/dn.rb index ea88dedadf5..a188aa168c1 100644 --- a/lib/gitlab/auth/ldap/dn.rb +++ b/lib/gitlab/auth/ldap/dn.rb @@ -30,7 +30,7 @@ module Gitlab def self.normalize_value(given_value) dummy_dn = "placeholder=#{given_value}" normalized_dn = new(*dummy_dn).to_normalized_s - normalized_dn.sub(/\Aplaceholder=/, '') + normalized_dn.delete_prefix('placeholder=') end ## diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index 41a8739b0b6..1a25ed10d81 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -5,6 +5,7 @@ module Gitlab module OAuth class Provider LABELS = { + "alicloud" => "AliCloud", "dingtalk" => "DingTalk", "github" => "GitHub", "gitlab" => "GitLab.com", diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb index b0a8c3a8cbb..52ff3aaa423 100644 --- a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb +++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb @@ -22,8 +22,6 @@ module Gitlab def perform(start_id, end_id) eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id).pluck(:id) - return if eligible_mrs.empty? - eligible_mrs.each_slice(10) do |slice| MergeRequest.where(id: slice).update_all(draft: true) end diff --git a/lib/gitlab/background_migration/backfill_group_features.rb b/lib/gitlab/background_migration/backfill_group_features.rb new file mode 100644 index 00000000000..084c788c8cb --- /dev/null +++ b/lib/gitlab/background_migration/backfill_group_features.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill group_features for an array of groups + class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BaseJob + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, batch_size) + pause_ms = 0 if pause_ms < 0 + + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size, order_hint: :type) do |sub_batch| + batch_metrics.time_operation(:upsert_group_features) do + upsert_group_features(sub_batch, batch_size) + end + + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(source_table, connection: connection) + .where(source_key_column => start_id..stop_id) + .where(type: 'Group') + end + + def upsert_group_features(relation, batch_size) + connection.execute( + <<~SQL + INSERT INTO group_features (group_id, created_at, updated_at) + SELECT namespaces.id as group_id, now(), now() + FROM namespaces + WHERE namespaces.type = 'Group' AND namespaces.id IN(#{relation.select(:id).limit(batch_size).to_sql}) + ON CONFLICT (group_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb b/lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb deleted file mode 100644 index 2d46ff6b933..00000000000 --- a/lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # BackfillIncidentIssueEscalationStatuses adds - # IncidentManagement::IssuableEscalationStatus records for existing Incident issues. - # They will be added with no policy, and escalations_started_at as nil. - class BackfillIncidentIssueEscalationStatuses - def perform(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO incident_management_issuable_escalation_statuses (issue_id, created_at, updated_at) - SELECT issues.id, current_timestamp, current_timestamp - FROM issues - WHERE issues.issue_type = 1 - AND issues.id BETWEEN #{start_id} AND #{stop_id} - ON CONFLICT (issue_id) DO NOTHING; - SQL - - mark_job_as_succeeded(start_id, stop_id) - end - - private - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb new file mode 100644 index 00000000000..1f0d606f001 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the `routes.namespace_id` column, by setting it to project.project_namespace_id + class BackfillNamespaceIdForProjectRoute + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + cleanup_gin_index('routes') + + batch_metrics.time_operation(:update_all) do + ActiveRecord::Base.connection.execute <<~SQL + WITH route_and_ns(route_id, project_namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + #{sub_batch.to_sql} + ) + UPDATE routes + SET namespace_id = route_and_ns.project_namespace_id + FROM route_and_ns + WHERE id = route_and_ns.route_id + SQL + end + + pause_ms = [0, pause_ms].max + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + def cleanup_gin_index(table_name) + sql = "select indexname::text from pg_indexes where tablename = '#{table_name}' and indexdef ilike '%gin%'" + index_names = ActiveRecord::Base.connection.select_values(sql) + + index_names.each do |index_name| + ActiveRecord::Base.connection.execute("select gin_clean_pending_list('#{index_name}')") + end + end + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(source_table, connection: ActiveRecord::Base.connection) + .joins('INNER JOIN projects ON routes.source_id = projects.id') + .where(source_key_column => start_id..stop_id) + .where(namespace_id: nil) + .where(source_type: 'Project') + .where.not(projects: { project_namespace_id: nil }) + .select("routes.id, projects.project_namespace_id") + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb new file mode 100644 index 00000000000..a16efa4222b --- /dev/null +++ b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the `issues.work_item_type_id` column, replacing any + # instances of `NULL` with the appropriate `work_item_types.id` based on `issues.issue_type` + class BackfillWorkItemTypeIdForIssues + # Basic AR model for issues table + class MigrationIssue < ApplicationRecord + include ::EachBatch + + self.table_name = 'issues' + + scope :base_query, ->(base_type) { where(work_item_type_id: nil, issue_type: base_type) } + end + + MAX_UPDATE_RETRIES = 3 + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, base_type, base_type_id) + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id, base_type) + + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + + # The query need to be reconstructed because .each_batch modifies the default scope + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 + reconstructed_sub_batch = MigrationIssue.unscoped.base_query(base_type).where(id: first..last) + + batch_metrics.time_operation(:update_all) do + update_with_retry(reconstructed_sub_batch, base_type_id) + end + + pause_ms = 0 if pause_ms < 0 + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + # Retry mechanism required as update statements on the issues table will randomly take longer than + # expected due to gin indexes https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71869#note_775796352 + def update_with_retry(sub_batch, base_type_id) + update_attempt = 1 + + begin + update_batch(sub_batch, base_type_id) + rescue ActiveRecord::StatementTimeout, ActiveRecord::QueryCanceled => e + update_attempt += 1 + + if update_attempt <= MAX_UPDATE_RETRIES + # sleeping 30 seconds as it might take a long time to clean the gin index pending list + sleep(30) + retry + end + + raise e + end + end + + def update_batch(sub_batch, base_type_id) + sub_batch.update_all(work_item_type_id: base_type_id) + end + + def relation_scoped_to_range(source_table, source_key_column, start_id, end_id, base_type) + MigrationIssue.where(source_key_column => start_id..end_id).base_query(base_type) + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb new file mode 100644 index 00000000000..06036eebcb9 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Batching class to use for back-filling issue's work_item_type_id for a single issue type. + # Batches will be scoped to records where the foreign key is NULL and only of a given issue type + # + # If no more batches exist in the table, returns nil. + class BackfillIssueWorkItemTypeBatchingStrategy < PrimaryKeyBatchingStrategy + def apply_additional_filters(relation, job_arguments:) + issue_type = job_arguments.first + + relation.where(issue_type: issue_type) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb index 5569bac0e19..e7a68b183b8 100644 --- a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb @@ -23,6 +23,7 @@ module Gitlab quoted_column_name = model_class.connection.quote_column_name(column_name) relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) + relation = apply_additional_filters(relation, job_arguments: job_arguments) next_batch_bounds = nil relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop @@ -33,6 +34,22 @@ module Gitlab next_batch_bounds end + + # Strategies based on PrimaryKeyBatchingStrategy can use + # this method to easily apply additional filters. + # + # Example: + # + # class MatchingType < PrimaryKeyBatchingStrategy + # def apply_additional_filters(relation, job_arguments:) + # type = job_arguments.first + # + # relation.where(type: type) + # end + # end + def apply_additional_filters(relation, job_arguments: []) + relation + end end end end diff --git a/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb b/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb new file mode 100644 index 00000000000..b703faf6a6c --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Cleanup draft column data inserted by a faulty regex + # + class CleanupDraftDataFromFaultyRegex + # Migration only version of MergeRequest table + ## + class MergeRequest < ActiveRecord::Base + LEAKY_REGEXP_STR = "^\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP" + CORRECTED_REGEXP_STR = "^(\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP)" + + include EachBatch + + self.table_name = 'merge_requests' + + def self.eligible + where(state_id: 1) + .where(draft: true) + .where("title ~* ?", LEAKY_REGEXP_STR) + .where("title !~* ?", CORRECTED_REGEXP_STR) + end + end + + def perform(start_id, end_id) + eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id).pluck(:id) + + return if eligible_mrs.empty? + + eligible_mrs.each_slice(10) do |slice| + MergeRequest.where(id: slice).update_all(draft: false) + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'CleanupDraftDataFromFaultyRegex', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/encrypt_static_object_token.rb b/lib/gitlab/background_migration/encrypt_static_object_token.rb index 80931353e2f..a087d2529eb 100644 --- a/lib/gitlab/background_migration/encrypt_static_object_token.rb +++ b/lib/gitlab/background_migration/encrypt_static_object_token.rb @@ -52,9 +52,9 @@ module Gitlab WHERE cte_id = id SQL end - - mark_job_as_succeeded(start_id, end_id) end + + mark_job_as_succeeded(start_id, end_id) end private diff --git a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb b/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb new file mode 100644 index 00000000000..defd9ea832b --- /dev/null +++ b/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Fix project name duplicates and backfill missing project namespace ids + class FixDuplicateProjectNameAndPath + SUB_BATCH_SIZE = 10 + # isolated project active record + class Project < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'projects' + + scope :without_project_namespace, -> { where(project_namespace_id: nil) } + scope :id_in, ->(ids) { where(id: ids) } + end + + def perform(start_id, end_id) + @project_ids = fetch_project_ids(start_id, end_id) + backfill_project_namespaces_service = init_backfill_service(project_ids) + backfill_project_namespaces_service.cleanup_gin_index('projects') + + project_ids.each_slice(SUB_BATCH_SIZE) do |ids| + ActiveRecord::Base.connection.execute(update_projects_name_and_path_sql(ids)) + end + + backfill_project_namespaces_service.backfill_project_namespaces + + mark_job_as_succeeded(start_id, end_id) + end + + private + + attr_accessor :project_ids + + def fetch_project_ids(start_id, end_id) + Project.without_project_namespace.where(id: start_id..end_id) + end + + def init_backfill_service(project_ids) + service = Gitlab::BackgroundMigration::ProjectNamespaces::BackfillProjectNamespaces.new + service.project_ids = project_ids + service.sub_batch_size = SUB_BATCH_SIZE + + service + end + + def update_projects_name_and_path_sql(project_ids) + <<~SQL + WITH cte (project_id, path_from_route ) AS ( + #{path_from_route_sql(project_ids).to_sql} + ) + UPDATE + projects + SET + name = concat(projects.name, '-', id), + path = CASE + WHEN projects.path <> cte.path_from_route THEN path_from_route + ELSE projects.path + END + FROM + cte + WHERE + projects.id = cte.project_id; + SQL + end + + def path_from_route_sql(project_ids) + Project.without_project_namespace.id_in(project_ids) + .joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'") + .select("projects.id, SUBSTRING(routes.path FROM '[^/]+(?=/$|$)') AS path_from_route") + end + + def mark_job_as_succeeded(*arguments) + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'FixDuplicateProjectNameAndPath', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/merge_topics_with_same_name.rb b/lib/gitlab/background_migration/merge_topics_with_same_name.rb new file mode 100644 index 00000000000..07231098a5f --- /dev/null +++ b/lib/gitlab/background_migration/merge_topics_with_same_name.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to merge project topics with the same case insensitive name + class MergeTopicsWithSameName + # Temporary AR model for topics + class Topic < ActiveRecord::Base + self.table_name = 'topics' + end + + # Temporary AR model for project topic assignment + class ProjectTopic < ActiveRecord::Base + self.table_name = 'project_topics' + end + + def perform(topic_names) + topic_names.each do |topic_name| + topics = Topic.where('LOWER(name) = ?', topic_name) + .order(total_projects_count: :desc, non_private_projects_count: :desc, id: :asc) + .to_a + topic_to_keep = topics.shift + merge_topics(topic_to_keep, topics) if topics.any? + end + end + + private + + def merge_topics(topic_to_keep, topics_to_remove) + description = topic_to_keep.description + + topics_to_remove.each do |topic| + description ||= topic.description if topic.description.present? + process_avatar(topic_to_keep, topic) if topic.avatar.present? + + ProjectTopic.transaction do + ProjectTopic.where(topic_id: topic.id) + .where.not(project_id: ProjectTopic.where(topic_id: topic_to_keep).select(:project_id)) + .update_all(topic_id: topic_to_keep.id) + ProjectTopic.where(topic_id: topic.id).delete_all + end + end + + Topic.where(id: topics_to_remove).delete_all + + topic_to_keep.update( + description: description, + total_projects_count: total_projects_count(topic_to_keep.id), + non_private_projects_count: non_private_projects_count(topic_to_keep.id) + ) + end + + # We intentionally use application code here because we need to copy/remove avatar files + def process_avatar(topic_to_keep, topic_to_remove) + topic_to_remove = ::Projects::Topic.find(topic_to_remove.id) + topic_to_keep = ::Projects::Topic.find(topic_to_keep.id) + unless topic_to_keep.avatar.present? + topic_to_keep.avatar = topic_to_remove.avatar + topic_to_keep.save! + end + + topic_to_remove.remove_avatar! + topic_to_remove.save! + end + + def total_projects_count(topic_id) + ProjectTopic.where(topic_id: topic_id).count + end + + def non_private_projects_count(topic_id) + ProjectTopic.joins('INNER JOIN projects ON project_topics.project_id = projects.id') + .where(project_topics: { topic_id: topic_id }).where('projects.visibility_level in (10, 20)').count + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb b/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb new file mode 100644 index 00000000000..ec4631d1e34 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to migrate category of integrations to third_party_wiki for confluence and shimo + class MigrateShimoConfluenceIntegrationCategory + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id) + define_batchable_model('integrations', connection: ::ActiveRecord::Base.connection) + .where(id: start_id..end_id, type_new: %w[Integrations::Confluence Integrations::Shimo]) + .update_all(category: 'third_party_wiki') + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb b/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb new file mode 100644 index 00000000000..9e102ea1517 --- /dev/null +++ b/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to populates the migration_plan column of container_repositories + # with the current plan of the namespaces that owns the container_repository + # + # The plan can be NULL, in which case no UPDATE + # will be executed. + class PopulateContainerRepositoryMigrationPlan + def perform(start_id, end_id) + (start_id..end_id).each do |id| + execute(<<~SQL) + WITH selected_plan AS ( + SELECT "plans"."name" + FROM "container_repositories" + INNER JOIN "projects" ON "projects"."id" = "container_repositories"."project_id" + INNER JOIN "namespaces" ON "namespaces"."id" = "projects"."namespace_id" + INNER JOIN "gitlab_subscriptions" ON "gitlab_subscriptions"."namespace_id" = "namespaces"."traversal_ids"[1] + INNER JOIN "plans" ON "plans"."id" = "gitlab_subscriptions"."hosted_plan_id" + WHERE "container_repositories"."id" = #{id} + ) + UPDATE container_repositories + SET migration_plan = selected_plan.name + FROM selected_plan + WHERE container_repositories.id = #{id}; + SQL + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def connection + @connection ||= ::ActiveRecord::Base.connection + end + + def execute(sql) + connection.execute(sql) + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_namespace_statistics.rb b/lib/gitlab/background_migration/populate_namespace_statistics.rb index e873ad412f2..97927ef48c2 100644 --- a/lib/gitlab/background_migration/populate_namespace_statistics.rb +++ b/lib/gitlab/background_migration/populate_namespace_statistics.rb @@ -5,9 +5,40 @@ module Gitlab # This class creates/updates those namespace statistics # that haven't been created nor initialized. # It also updates the related namespace statistics - # This is only required in EE class PopulateNamespaceStatistics def perform(group_ids, statistics) + # Updating group statistics might involve calling Gitaly. + # For example, when calculating `wiki_size`, we will need + # to perform the request to check if the repo exists and + # also the repository size. + # + # The `allow_n_plus_1_calls` method is only intended for + # dev and test. It won't be raised in prod. + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + relation(group_ids).each do |group| + upsert_namespace_statistics(group, statistics) + end + end + end + + private + + def upsert_namespace_statistics(group, statistics) + response = ::Groups::UpdateStatisticsService.new(group, statistics: statistics).execute + + error_message("#{response.message} group: #{group.id}") if response.error? + end + + def logger + @logger ||= ::Gitlab::BackgroundMigration::Logger.build + end + + def error_message(message) + logger.error(message: "Namespace Statistics Migration: #{message}") + end + + def relation(group_ids) + Group.includes(:namespace_statistics).where(id: group_ids) end end end diff --git a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb index c34cc57ce60..bd7d7d02162 100644 --- a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb +++ b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb @@ -7,6 +7,8 @@ module Gitlab # # rubocop: disable Metrics/ClassLength class BackfillProjectNamespaces + attr_accessor :project_ids, :sub_batch_size + SUB_BATCH_SIZE = 25 PROJECT_NAMESPACE_STI_NAME = 'Project' @@ -18,7 +20,7 @@ module Gitlab case migration_type when 'up' - backfill_project_namespaces(namespace_id) + backfill_project_namespaces mark_job_as_succeeded(start_id, end_id, namespace_id, 'up') when 'down' cleanup_backfilled_project_namespaces(namespace_id) @@ -28,11 +30,7 @@ module Gitlab end end - private - - attr_accessor :project_ids, :sub_batch_size - - def backfill_project_namespaces(namespace_id) + def backfill_project_namespaces project_ids.each_slice(sub_batch_size) do |project_ids| # cleanup gin indexes on namespaces table cleanup_gin_index('namespaces') @@ -64,6 +62,8 @@ module Gitlab end end + private + def cleanup_backfilled_project_namespaces(namespace_id) project_ids.each_slice(sub_batch_size) do |project_ids| # IMPORTANT: first nullify project_namespace_id in projects table to avoid removing projects when records diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index 78a8f39e143..e210c18e3d1 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -2,11 +2,16 @@ module Gitlab class Blame - attr_accessor :blob, :commit + attr_accessor :blob, :commit, :range - def initialize(blob, commit) + def initialize(blob, commit, range: nil) @blob = blob @commit = commit + @range = range + end + + def first_line + range&.first || 1 end def groups(highlight: true) @@ -14,14 +19,14 @@ module Gitlab groups = [] current_group = nil - i = 0 - blame.each do |commit, line| + i = first_line - 1 + blame.each do |commit, line, previous_path| commit = Commit.new(commit, project) commit.lazy_author # preload author if prev_sha != commit.sha groups << current_group if current_group - current_group = { commit: commit, lines: [] } + current_group = { commit: commit, lines: [], previous_path: previous_path } end current_group[:lines] << (highlight ? highlighted_lines[i].html_safe : line) @@ -37,7 +42,7 @@ module Gitlab private def blame - @blame ||= Gitlab::Git::Blame.new(repository, @commit.id, @blob.path) + @blame ||= Gitlab::Git::Blame.new(repository, @commit.id, @blob.path, range: range) end def highlighted_lines diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index ef936581c10..10233cf4228 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -447,9 +447,8 @@ module Gitlab end def state - state = STATE_PARAMS.inject({}) do |h, param| + state = STATE_PARAMS.each_with_object({}) do |param, h| h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend - h end Base64.urlsafe_encode64(state.to_json) end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 2b190d89fa4..2c9524c89ff 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -6,6 +6,8 @@ module Gitlab # Base GitLab CI Configuration facade # class Config + include Gitlab::Utils::StrongMemoize + ConfigError = Class.new(StandardError) TIMEOUT_SECONDS = 30.seconds TIMEOUT_MESSAGE = 'Resolving config took longer than expected' @@ -22,6 +24,11 @@ module Gitlab def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, logger: nil) @logger = logger || ::Gitlab::Ci::Pipeline::Logger.new(project: project) @source_ref_path = pipeline&.source_ref_path + @project = project + + if use_config_variables? + pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) + end @context = self.logger.instrument(:config_build_context) do build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) @@ -82,7 +89,13 @@ module Gitlab end def included_templates - @context.expandset.filter_map { |i| i[:template] } + @context.includes.filter_map { |i| i[:location] if i[:type] == :template } + end + + def metadata + { + includes: @context.includes + } end private @@ -149,6 +162,10 @@ module Gitlab end def build_variables_without_instrumentation(project:, pipeline:) + if use_config_variables? + return pipeline.variables_builder.config_variables + end + Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless project @@ -178,6 +195,12 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, @context.sentry_payload) end + def use_config_variables? + strong_memoize(:use_config_variables) do + ::Feature.enabled?(:ci_variables_builder_config_variables, @project, default_enabled: :yaml) + end + end + # Overridden in EE def rescue_errors RESCUE_ERRORS diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 43475742214..46afedbcc3a 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -23,7 +23,7 @@ module Gitlab validates :config, presence: true validates :name, presence: true validates :name, type: Symbol - validates :name, length: { maximum: 255 }, if: -> { ::Feature.enabled?(:ci_validate_job_length, default_enabled: :yaml) } + validates :name, length: { maximum: 255 } validates :config, disallowed_keys: { in: %i[only except start_in], diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 512cfdde474..2def565bc19 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -70,16 +70,20 @@ module Gitlab } end - def mask_variables_from(location) - variables.reduce(location.dup) do |loc, variable| + def mask_variables_from(string) + variables.reduce(string.dup) do |str, variable| if variable[:masked] - Gitlab::Ci::MaskSecret.mask!(loc, variable[:value]) + Gitlab::Ci::MaskSecret.mask!(str, variable[:value]) else - loc + str end end end + def includes + expandset.map(&:metadata) + end + protected attr_writer :expandset, :execution_deadline, :logger diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 4f79e64ca9a..1244c7f7475 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -28,6 +28,14 @@ module Gitlab end end + def metadata + super.merge( + type: :artifact, + location: masked_location, + extra: { job_name: masked_job_name } + ) + end + private def project @@ -52,7 +60,7 @@ module Gitlab end unless artifact_job.present? - errors.push("Job `#{job_name}` not found in parent pipeline or does not have artifacts!") + errors.push("Job `#{masked_job_name}` not found in parent pipeline or does not have artifacts!") return false end @@ -80,6 +88,12 @@ module Gitlab parent_pipeline: context.parent_pipeline } end + + def masked_job_name + strong_memoize(:masked_job_name) do + context.mask_variables_from(job_name) + end + end end end end diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index a660dd339d8..89da0796906 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -16,8 +16,6 @@ module Gitlab @params = params @context = context @errors = [] - - validate! end def matching? @@ -48,6 +46,30 @@ module Gitlab expanded_content_hash end + def validate! + context.logger.instrument(:config_file_validation) do + validate_execution_time! + validate_location! + validate_content! if errors.none? + validate_hash! if errors.none? + end + end + + def metadata + { + context_project: context.project&.full_path, + context_sha: context.sha + } + end + + def eql?(other) + other.hash == hash + end + + def hash + [params, context.project&.full_path, context.sha].hash + end + protected def expanded_content_hash @@ -66,13 +88,6 @@ module Gitlab nil end - def validate! - validate_execution_time! - validate_location! - validate_content! if errors.none? - validate_hash! if errors.none? - end - def validate_execution_time! context.check_execution_time! end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index 3aa665c7d18..ee9cc1552fe 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -19,6 +19,14 @@ module Gitlab strong_memoize(:content) { fetch_local_content } end + def metadata + super.merge( + type: :local, + location: masked_location, + extra: {} + ) + end + private def validate_content! diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index 27e097ba980..3d4436530a8 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -27,17 +27,25 @@ module Gitlab strong_memoize(:content) { fetch_local_content } end + def metadata + super.merge( + type: :file, + location: masked_location, + extra: { project: masked_project_name, ref: masked_ref_name } + ) + end + private def validate_content! if !can_access_local_content? - errors.push("Project `#{project_name}` not found or access denied!") + errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.") elsif sha.nil? - errors.push("Project `#{project_name}` reference `#{ref_name}` does not exist!") + errors.push("Project `#{masked_project_name}` reference `#{masked_ref_name}` does not exist!") elsif content.nil? - errors.push("Project `#{project_name}` file `#{masked_location}` does not exist!") + errors.push("Project `#{masked_project_name}` file `#{masked_location}` does not exist!") elsif content.blank? - errors.push("Project `#{project_name}` file `#{masked_location}` is empty!") + errors.push("Project `#{masked_project_name}` file `#{masked_location}` is empty!") end end @@ -76,6 +84,18 @@ module Gitlab variables: context.variables } end + + def masked_project_name + strong_memoize(:masked_project_name) do + context.mask_variables_from(project_name) + end + end + + def masked_ref_name + strong_memoize(:masked_ref_name) do + context.mask_variables_from(ref_name) + end + end end end end diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index 8335a9ef625..e7b007b4d8d 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -18,6 +18,14 @@ module Gitlab strong_memoize(:content) { fetch_remote_content } end + def metadata + super.merge( + type: :remote, + location: masked_location, + extra: {} + ) + end + private def validate_location! diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index c3d120dfdce..9469f09ce13 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -20,6 +20,14 @@ module Gitlab strong_memoize(:content) { fetch_template_content } end + def metadata + super.merge( + type: :template, + location: masked_location, + extra: {} + ) + end + private def validate_location! diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 79a04ad409e..c1250c82750 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -48,8 +48,8 @@ module Gitlab .flat_map(&method(:expand_project_files)) .flat_map(&method(:expand_wildcard_paths)) .map(&method(:expand_variables)) - .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) + .each(&method(:verify!)) end def normalize_location(location) @@ -111,26 +111,6 @@ module Gitlab end end - def verify_duplicates!(location) - logger.instrument(:config_mapper_verify) do - verify_max_includes_and_add_location!(location) - end - end - - def verify_max_includes_and_add_location!(location) - if expandset.count >= MAX_INCLUDES - raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!" - end - - # Scope location to context to allow support of - # relative includes - scoped_location = location.merge( - context_project: context.project, - context_sha: context.sha) - - expandset.add(scoped_location) - end - def select_first_matching(location) logger.instrument(:config_mapper_select) do select_first_matching_without_instrumentation(location) @@ -147,6 +127,18 @@ module Gitlab matching.first end + def verify!(location_object) + verify_max_includes! + location_object.validate! + expandset.add(location_object) + end + + def verify_max_includes! + if expandset.count >= MAX_INCLUDES + raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!" + end + end + def expand_variables(data) logger.instrument(:config_mapper_variables) do expand_variables_without_instrumentation(data) diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 7baae2f53d7..13a159f3745 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -14,6 +14,7 @@ module Gitlab def initialize(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false) @json_data = json_data @report = report + @project = report.project @validate = validate @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled end @@ -43,31 +44,41 @@ module Gitlab attr_reader :json_data, :report, :validate def valid? - if Feature.enabled?(:show_report_validation_warnings, default_enabled: :yaml) - # We want validation to happen regardless of VALIDATE_SCHEMA CI variable + # We want validation to happen regardless of VALIDATE_SCHEMA + # CI variable. + # + # Previously it controlled BOTH validation and enforcement of + # schema validation result. + # + # After 15.0 we will enforce schema validation by default + # See: https://gitlab.com/groups/gitlab-org/-/epics/6968 + schema_validator.deprecation_warnings.each { |deprecation_warning| report.add_warning('Schema', deprecation_warning) } + + if validate schema_validation_passed = schema_validator.valid? - if validate - schema_validator.errors.each { |error| report.add_error('Schema', error) } unless schema_validation_passed - - schema_validation_passed - else - # We treat all schema validation errors as warnings - schema_validator.errors.each { |error| report.add_warning('Schema', error) } + # Validation warnings are errors + schema_validator.errors.each { |error| report.add_error('Schema', error) } + schema_validator.warnings.each { |warning| report.add_error('Schema', warning) } - true - end + schema_validation_passed else - return true if !validate || schema_validator.valid? + # Validation warnings are warnings + schema_validator.errors.each { |error| report.add_warning('Schema', error) } + schema_validator.warnings.each { |warning| report.add_warning('Schema', warning) } - schema_validator.errors.each { |error| report.add_error('Schema', error) } - - false + true end end def schema_validator - @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data, report.version) + @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new( + report.type, + report_data, + report.version, + project: @project, + scanner: top_level_scanner + ) end def report_data @@ -137,7 +148,7 @@ module Gitlab metadata_version: report_version, details: data['details'] || {}, signatures: signatures, - project_id: report.project_id, + project_id: @project.id, vulnerability_finding_signatures_enabled: @vulnerability_finding_signatures_enabled)) end @@ -280,7 +291,7 @@ module Gitlab report_type: report.type, primary_identifier_fingerprint: primary_identifier&.fingerprint, location_fingerprint: location_fingerprint, - project_id: report.project_id + project_id: @project.id } if uuid_v5_name_components.values.any?(&:nil?) diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 0ab1a128052..cef029bd749 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -8,14 +8,14 @@ module Gitlab class SchemaValidator # https://docs.gitlab.com/ee/update/deprecations.html#147 SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0], - container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], - coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], - dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], - api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], - dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], - sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], - secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0] + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1] }.freeze # https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/tags @@ -26,19 +26,19 @@ module Gitlab 8.0.0-rc1 8.0.1-rc1 8.1.0-rc1 9.0.0-rc1].freeze # These come from https://app.periscopedata.com/app/gitlab/895813/Secure-Scan-metrics?widget=12248944&udv=1385516 - KNOWN_VERSIONS_TO_DEPRECATE = %w[0.1 1.0 1.0.0 1.2 1.3 10.0.0 12.1.0 13.1.0 2.0 2.1 2.1.0 2.3 2.3.0 2.4 3.0 3.0.0 3.0.6 3.13.2 V2.7.0].freeze + KNOWN_VERSIONS_TO_REMOVE = %w[0.1 1.0 1.0.0 1.2 1.3 10.0.0 12.1.0 13.1.0 2.0 2.1 2.1.0 2.3 2.3.0 2.4 3.0 3.0.0 3.0.6 3.13.2 V2.7.0].freeze - VERSIONS_TO_DEPRECATE_IN_15_0 = (PREVIOUS_RELEASES + KNOWN_VERSIONS_TO_DEPRECATE).freeze + VERSIONS_TO_REMOVE_IN_15_0 = (PREVIOUS_RELEASES + KNOWN_VERSIONS_TO_REMOVE).freeze DEPRECATED_VERSIONS = { - cluster_image_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, - container_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, - coverage_fuzzing: VERSIONS_TO_DEPRECATE_IN_15_0, - dast: VERSIONS_TO_DEPRECATE_IN_15_0, - api_fuzzing: VERSIONS_TO_DEPRECATE_IN_15_0, - dependency_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, - sast: VERSIONS_TO_DEPRECATE_IN_15_0, - secret_detection: VERSIONS_TO_DEPRECATE_IN_15_0 + cluster_image_scanning: VERSIONS_TO_REMOVE_IN_15_0, + container_scanning: VERSIONS_TO_REMOVE_IN_15_0, + coverage_fuzzing: VERSIONS_TO_REMOVE_IN_15_0, + dast: VERSIONS_TO_REMOVE_IN_15_0, + api_fuzzing: VERSIONS_TO_REMOVE_IN_15_0, + dependency_scanning: VERSIONS_TO_REMOVE_IN_15_0, + sast: VERSIONS_TO_REMOVE_IN_15_0, + secret_detection: VERSIONS_TO_REMOVE_IN_15_0 }.freeze class Schema @@ -86,20 +86,110 @@ module Gitlab end end - def initialize(report_type, report_data, report_version = nil) - @report_type = report_type + def initialize(report_type, report_data, report_version = nil, project: nil, scanner: nil) + @report_type = report_type&.to_sym @report_data = report_data @report_version = report_version + @project = project + @scanner = scanner + @errors = [] + @warnings = [] + @deprecation_warnings = [] + + populate_errors + populate_warnings + populate_deprecation_warnings end def valid? errors.empty? end - def errors - @errors ||= schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) } + def populate_errors + schema_validation_errors = schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) } + + log_warnings(problem_type: 'schema_validation_fails') unless schema_validation_errors.empty? + + if Feature.enabled?(:enforce_security_report_validation, @project) + @errors += schema_validation_errors + else + @warnings += schema_validation_errors + end + end + + def populate_warnings + add_unsupported_report_version_message if !report_uses_supported_schema_version? && !report_uses_deprecated_schema_version? + end + + def populate_deprecation_warnings + add_deprecated_report_version_message if report_uses_deprecated_schema_version? + end + + def add_deprecated_report_version_message + log_warnings(problem_type: 'using_deprecated_schema_version') + + message = "Version #{report_version} for report type #{report_type} has been deprecated, supported versions for this report type are: #{supported_schema_versions}" + add_message_as(level: :deprecation_warning, message: message) + end + + def log_warnings(problem_type:) + Gitlab::AppLogger.info( + message: 'security report schema validation problem', + security_report_type: report_type, + security_report_version: report_version, + project_id: @project.id, + security_report_failure: problem_type, + security_report_scanner_id: @scanner&.dig('id'), + security_report_scanner_version: @scanner&.dig('version') + ) + end + + def add_unsupported_report_version_message + log_warnings(problem_type: 'using_unsupported_schema_version') + + if Feature.enabled?(:enforce_security_report_validation, @project) + handle_unsupported_report_version(treat_as: :error) + else + handle_unsupported_report_version(treat_as: :warning) + end + end + + def report_uses_deprecated_schema_version? + DEPRECATED_VERSIONS[report_type].include?(report_version) + end + + def report_uses_supported_schema_version? + SUPPORTED_VERSIONS[report_type].include?(report_version) end + def handle_unsupported_report_version(treat_as:) + if report_version.nil? + message = "Report version not provided, #{report_type} report type supports versions: #{supported_schema_versions}" + add_message_as(level: treat_as, message: message) + else + message = "Version #{report_version} for report type #{report_type} is unsupported, supported versions for this report type are: #{supported_schema_versions}" + end + + add_message_as(level: treat_as, message: message) + end + + def supported_schema_versions + SUPPORTED_VERSIONS[report_type].join(", ") + end + + def add_message_as(level:, message:) + case level + when :deprecation_warning + @deprecation_warnings << message + when :error + @errors << message + when :warning + @warnings << message + end + end + + attr_reader :errors, :warnings, :deprecation_warnings + private attr_reader :report_type, :report_data, :report_version diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..7bcb2d5867f --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/cluster-image-scanning-report-format.json @@ -0,0 +1,977 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/container-scanning-report-format.json new file mode 100644 index 00000000000..a13e0418499 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/container-scanning-report-format.json @@ -0,0 +1,911 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+$", + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..050c34669b3 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/coverage-fuzzing-report-format.json @@ -0,0 +1,874 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dast-report-format.json new file mode 100644 index 00000000000..62ed293ad44 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dast-report-format.json @@ -0,0 +1,1291 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "minLength": 1, + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "minLength": 1, + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "minLength": 1, + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "minLength": 1, + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + }, + "discovered_at": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", + "examples": [ + "2020-01-28T03:26:02.956" + ] + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dependency-scanning-report-format.json new file mode 100644 index 00000000000..1e3f4188845 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dependency-scanning-report-format.json @@ -0,0 +1,968 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "dependency_files", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/sast-report-format.json new file mode 100644 index 00000000000..4c57d20dbaa --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/sast-report-format.json @@ -0,0 +1,869 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/secret-detection-report-format.json new file mode 100644 index 00000000000..b1337954e97 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/secret-detection-report-format.json @@ -0,0 +1,892 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb b/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb new file mode 100644 index 00000000000..cb02f09f819 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Limit + class RateLimit < Chain::Base + include Chain::Helpers + + def perform! + return unless throttle_enabled? + + # We exclude child-pipelines from the rate limit because they represent + # sub-pipelines that would otherwise hit the rate limit due to having the + # same scope (project, user, sha). + # + return if pipeline.parent_pipeline? + + if rate_limit_throttled? + create_log_entry + error(throttle_message) unless dry_run? + end + end + + def break? + @pipeline.errors.any? + end + + private + + def rate_limit_throttled? + ::Gitlab::ApplicationRateLimiter.throttled?( + :pipelines_create, scope: [project, current_user, command.sha] + ) + end + + def create_log_entry + Gitlab::AppJsonLogger.info( + class: self.class.name, + namespace_id: project.namespace_id, + project_id: project.id, + commit_sha: command.sha, + current_user_id: current_user.id, + subscription_plan: project.actual_plan_name, + message: 'Activated pipeline creation rate limit' + ) + end + + def throttle_message + 'Too many pipelines created in the last minute. Try again later.' + end + + def throttle_enabled? + ::Feature.enabled?( + :ci_throttle_pipelines_creation, + project, + default_enabled: :yaml) + end + + def dry_run? + ::Feature.enabled?( + :ci_throttle_pipelines_creation_dry_run, + project, + default_enabled: :yaml) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/template_usage.rb b/lib/gitlab/ci/pipeline/chain/template_usage.rb index 2fcf1740b5f..f9b3b6cd644 100644 --- a/lib/gitlab/ci/pipeline/chain/template_usage.rb +++ b/lib/gitlab/ci/pipeline/chain/template_usage.rb @@ -19,7 +19,7 @@ module Gitlab def track_event(template) Gitlab::UsageDataCounters::CiTemplateUniqueCounter - .track_unique_project_event(project_id: pipeline.project_id, template: template, config_source: pipeline.config_source) + .track_unique_project_event(project: pipeline.project, template: template, config_source: pipeline.config_source, user: current_user) end def included_templates diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb index 8c528056d0c..70f2919d38d 100644 --- a/lib/gitlab/ci/reports/security/report.rb +++ b/lib/gitlab/ci/reports/security/report.rb @@ -9,6 +9,7 @@ module Gitlab attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version, :schema_validation_status, :warnings delegate :project_id, to: :pipeline + delegate :project, to: :pipeline def initialize(type, pipeline, created_at) @type = type @@ -38,6 +39,10 @@ module Gitlab errors.present? end + def warnings? + warnings.present? + end + def add_scanner(scanner) scanners[scanner.key] ||= scanner end diff --git a/lib/gitlab/ci/reports/security/scanner.rb b/lib/gitlab/ci/reports/security/scanner.rb index c1de03cea44..1ac66a0c671 100644 --- a/lib/gitlab/ci/reports/security/scanner.rb +++ b/lib/gitlab/ci/reports/security/scanner.rb @@ -12,6 +12,7 @@ module Gitlab "gemnasium-maven" => 3, "gemnasium-python" => 3, "bandit" => 1, + "spotbugs" => 1, "semgrep" => 2 }.freeze diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index 00920dfbd54..d0388c65f58 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -12,7 +12,6 @@ module Gitlab def initialize(name = nil) @name = name @test_cases = {} - @all_test_cases = [] @total_time = 0.0 end diff --git a/lib/gitlab/ci/runner_releases.rb b/lib/gitlab/ci/runner_releases.rb new file mode 100644 index 00000000000..944c24ca128 --- /dev/null +++ b/lib/gitlab/ci/runner_releases.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class RunnerReleases + include Singleton + + RELEASES_VALIDITY_PERIOD = 1.day + RELEASES_VALIDITY_AFTER_ERROR_PERIOD = 5.seconds + + INITIAL_BACKOFF = 5.seconds + MAX_BACKOFF = 1.hour + BACKOFF_GROWTH_FACTOR = 2.0 + + def initialize + reset! + end + + # Returns a sorted list of the publicly available GitLab Runner releases + # + def releases + return @releases unless Time.now.utc >= @expire_time + + @releases = fetch_new_releases + end + + def reset! + @expire_time = Time.now.utc + @releases = nil + @backoff_count = 0 + end + + public_class_method :instance + + private + + def fetch_new_releases + response = Gitlab::HTTP.try_get(::Gitlab::CurrentSettings.current_application_settings.public_runner_releases_url) + + releases = response.success? ? extract_releases(response) : nil + ensure + @expire_time = (releases ? RELEASES_VALIDITY_PERIOD : next_backoff).from_now + end + + def extract_releases(response) + response.parsed_response.map { |release| parse_runner_release(release) }.sort! + end + + def parse_runner_release(release) + ::Gitlab::VersionInfo.parse(release['name'].delete_prefix('v')) + end + + def next_backoff + return MAX_BACKOFF if @backoff_count >= 11 # optimization to prevent expensive exponentiation and possible overflows + + backoff = (INITIAL_BACKOFF * (BACKOFF_GROWTH_FACTOR**@backoff_count)) + .clamp(INITIAL_BACKOFF, MAX_BACKOFF) + .seconds + @backoff_count += 1 + + backoff + end + end + end +end diff --git a/lib/gitlab/ci/runner_upgrade_check.rb b/lib/gitlab/ci/runner_upgrade_check.rb new file mode 100644 index 00000000000..baf041fc358 --- /dev/null +++ b/lib/gitlab/ci/runner_upgrade_check.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class RunnerUpgradeCheck + include Singleton + + def initialize + reset! + end + + def check_runner_upgrade_status(runner_version) + return :unknown unless runner_version + + releases = RunnerReleases.instance.releases + parsed_runner_version = runner_version.is_a?(::Gitlab::VersionInfo) ? runner_version : ::Gitlab::VersionInfo.parse(runner_version) + + raise ArgumentError, "'#{runner_version}' is not a valid version" unless parsed_runner_version.valid? + + available_releases = releases.reject { |release| release > @gitlab_version } + + return :recommended if available_releases.any? { |available_release| patch_update?(available_release, parsed_runner_version) } + return :recommended if outside_backport_window?(parsed_runner_version, releases) + return :available if available_releases.any? { |available_release| available_release > parsed_runner_version } + + :not_available + end + + def reset! + @gitlab_version = ::Gitlab::VersionInfo.parse(::Gitlab::VERSION) + end + + public_class_method :instance + + private + + def patch_update?(available_release, runner_version) + # https://docs.gitlab.com/ee/policy/maintenance.html#patch-releases + available_release.major == runner_version.major && + available_release.minor == runner_version.minor && + available_release.patch > runner_version.patch + end + + def outside_backport_window?(runner_version, releases) + return false if runner_version >= releases.last # return early if runner version is too new + + latest_minor_releases = releases.map { |r| version_without_patch(r) }.uniq { |v| v.to_s } + latest_version_position = latest_minor_releases.count - 1 + runner_version_position = latest_minor_releases.index(version_without_patch(runner_version)) + + return true if runner_version_position.nil? # consider outside if version is too old + + # https://docs.gitlab.com/ee/policy/maintenance.html#backporting-to-older-releases + latest_version_position - runner_version_position > 2 + end + + def version_without_patch(version) + ::Gitlab::VersionInfo.new(version.major, version.minor, 0) + end + end + end +end diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index df572188194..0074f3675e0 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -5,20 +5,36 @@ module Gitlab module Status module Build class Manual < Status::Extended + def self.matches?(build, user) + build.playable? + end + def illustration { image: 'illustrations/manual_action.svg', size: 'svg-394', title: _('This job requires a manual action'), - content: _('This job requires manual intervention to start. Before starting this job, you can add variables below for last-minute configuration changes.') + content: illustration_content } end - def self.matches?(build, user) - build.playable? + private + + def illustration_content + if can?(user, :update_build, subject) + _('This job requires manual intervention to start. Before starting this job, you can add variables below for last-minute configuration changes.') + else + generic_permission_failure_message + end + end + + def generic_permission_failure_message + _("This job does not run automatically and must be started manually, but you do not have access to it.") end end end end end end + +Gitlab::Ci::Status::Build::Manual.prepend_mod_with('Gitlab::Ci::Status::Build::Manual') diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml index bdcd3240380..c078c99f352 100644 --- a/lib/gitlab/ci/templates/C++.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -4,7 +4,7 @@ # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/C++.gitlab-ci.yml # use the official gcc image, based on debian -# can use verions as well, like gcc:5.2 +# can use versions as well, like gcc:5.2 # see https://hub.docker.com/_/gcc/ image: gcc diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index 19e4ffdbe1e..bd8e1020c4e 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -5,21 +5,6 @@ image: golang:latest -variables: - # Please edit to your GitLab project - REPO_NAME: gitlab.com/namespace/project - -# The problem is that to be able to use go get, one needs to put -# the repository in the $GOPATH. So for example if your gitlab domain -# is gitlab.com, and that your repository is namespace/project, and -# the default GOPATH being /go, then you'd need to have your -# repository in /go/src/gitlab.com/namespace/project -# Thus, making a symbolic link corrects this. -before_script: - - mkdir -p "$GOPATH/src/$(dirname $REPO_NAME)" - - ln -svf "$CI_PROJECT_DIR" "$GOPATH/src/$REPO_NAME" - - cd "$GOPATH/src/$REPO_NAME" - stages: - test - build @@ -35,7 +20,8 @@ format: compile: stage: build script: - - go build -race -ldflags "-extldflags '-static'" -o $CI_PROJECT_DIR/mybinary + - mkdir -p mybinaries + - go build -o mybinaries ./... artifacts: paths: - - mybinary + - mybinaries diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index cc204207f84..0cc5090f85e 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.23.0' .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index 1a99db67441..d41182ec9be 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -32,6 +32,16 @@ dependency_scanning: .ds-analyzer: extends: dependency_scanning allow_failure: true + variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/$DS_ANALYZER_NAME:$DS_MAJOR_VERSION" + # DS_ANALYZER_NAME is an undocumented variable used in job definitions + # to inject the analyzer name in the image name. + DS_ANALYZER_NAME: "" + image: + name: "$DS_ANALYZER_IMAGE$DS_IMAGE_SUFFIX" # `rules` must be overridden explicitly by each child job # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 script: @@ -46,13 +56,8 @@ gemnasium-dependency_scanning: extends: - .ds-analyzer - .cyclone-dx-reports - image: - name: "$DS_ANALYZER_IMAGE" variables: - # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium:$DS_MAJOR_VERSION" + DS_ANALYZER_NAME: "gemnasium" GEMNASIUM_LIBRARY_SCAN_ENABLED: "true" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -77,13 +82,8 @@ gemnasium-maven-dependency_scanning: extends: - .ds-analyzer - .cyclone-dx-reports - image: - name: "$DS_ANALYZER_IMAGE" variables: - # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" + DS_ANALYZER_NAME: "gemnasium-maven" # Stop reporting Gradle as "maven". # See https://gitlab.com/gitlab-org/gitlab/-/issues/338252 DS_REPORT_PACKAGE_MANAGER_MAVEN_WHEN_JAVA: "false" @@ -105,13 +105,8 @@ gemnasium-python-dependency_scanning: extends: - .ds-analyzer - .cyclone-dx-reports - image: - name: "$DS_ANALYZER_IMAGE" variables: - # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" + DS_ANALYZER_NAME: "gemnasium-python" # Stop reporting Pipenv and Setuptools as "pip". # See https://gitlab.com/gitlab-org/gitlab/-/issues/338252 DS_REPORT_PACKAGE_MANAGER_PIP_WHEN_PYTHON: "false" @@ -138,13 +133,8 @@ gemnasium-python-dependency_scanning: bundler-audit-dependency_scanning: extends: .ds-analyzer - image: - name: "$DS_ANALYZER_IMAGE" variables: - # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bundler-audit:$DS_MAJOR_VERSION" + DS_ANALYZER_NAME: "bundler-audit" rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never @@ -158,13 +148,8 @@ bundler-audit-dependency_scanning: retire-js-dependency_scanning: extends: .ds-analyzer - image: - name: "$DS_ANALYZER_IMAGE" variables: - # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/retire.js:$DS_MAJOR_VERSION" + DS_ANALYZER_NAME: "retire.js" rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index bc4f2099d94..89eb91c981f 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.23.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index ce584091eab..78f28b59aa5 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.23.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index 5ddfb2a54be..488e7ec72fd 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -1,7 +1,14 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/iac_scanning/ +# +# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/iac_scanning/index.html + variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SAST_IMAGE_SUFFIX: "" + SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" iac-sast: @@ -25,7 +32,7 @@ kics-iac-sast: name: "$SAST_ANALYZER_IMAGE" variables: SAST_ANALYZER_IMAGE_TAG: 1 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG" + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - if: $SAST_DISABLED when: never diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 8cc9ea0200c..7415fa3104c 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -7,6 +7,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SAST_IMAGE_SUFFIX: "" SAST_EXCLUDED_ANALYZERS: "" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" @@ -101,7 +102,11 @@ flawfinder-sast: - if: $CI_COMMIT_BRANCH exists: - '**/*.c' + - '**/*.cc' - '**/*.cpp' + - '**/*.c++' + - '**/*.cp' + - '**/*.cxx' kubesec-sast: extends: .sast-analyzer @@ -246,8 +251,9 @@ semgrep-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + SEARCH_MAX_DEPTH: 20 SAST_ANALYZER_IMAGE_TAG: 2 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG" + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - if: $SAST_DISABLED when: never @@ -262,6 +268,7 @@ semgrep-sast: - '**/*.tsx' - '**/*.c' - '**/*.go' + - '**/*.java' sobelow-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index 0ef6f63bb94..6aacd082fd7 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -6,12 +6,14 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECRET_DETECTION_IMAGE_SUFFIX: "" + SECRETS_ANALYZER_VERSION: "3" SECRET_DETECTION_EXCLUDED_PATHS: "" .secret-analyzer: stage: test - image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" + image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION$SECRET_DETECTION_IMAGE_SUFFIX" services: [] allow_failure: true variables: @@ -31,14 +33,7 @@ secret_detection: script: - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi # Historic scan - - | - if [ "$SECRET_DETECTION_HISTORIC_SCAN" == "true" ] - then - echo "historic scan" - git fetch --unshallow origin $CI_COMMIT_REF_NAME - /analyzer run - exit - fi + - if [ "$SECRET_DETECTION_HISTORIC_SCAN" == "true" ]; then echo "Running Secret Detection Historic Scan"; /analyzer run; exit; fi # Default branch scan - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit; fi # Push event diff --git a/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml b/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml new file mode 100644 index 00000000000..67c69115948 --- /dev/null +++ b/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml @@ -0,0 +1,96 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml + +# Use this template to run MATLAB and Simulink as part of your CI/CD pipeline. The template has three jobs: +# - `command`: Run MATLAB scripts, functions, and statements. +# - `test`: Run tests authored using the MATLAB unit testing framework or Simulink Test. +# - `test_artifacts_job`: Run MATLAB and Simulink tests, and generate test and coverage artifacts. +# +# You can copy and paste one or more jobs in this template into your `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# - To run MATLAB and Simulink, MATLAB must be installed on the runner that will run the jobs. +# The runner will use the topmost MATLAB version on the system path. +# The build fails if the operating system cannot find MATLAB on the path. +# - The jobs in this template use the `matlab -batch` syntax to start MATLAB. The `-batch` option is supported +# in MATLAB R2019a and later. + +# The `command` runs MATLAB scripts, functions, and statements. To use the job in your pipeline, +# substitute `command` with the code you want to run. +# +command: + script: matlab -batch command + +# If the value of `command` is the name of a MATLAB script or function, do not specify the file extension. +# For example, to run a script named `myscript.m` in the root of your repository, specify the `command` like this: +# +# "myscript" +# +# If you specify more than one script, function, or statement, use a comma or semicolon to separate them. +# For example, to run `myscript.m` in a folder named `myfolder` located in the root of the repository, +# you can specify the `command` like this: +# +# "addpath('myfolder'), myscript" +# +# MATLAB exits with exit code 0 if the specified script, function, or statement executes successfully without +# error. Otherwise, MATLAB terminates with a nonzero exit code, which causes the job to fail. To have the +# job fail in certain conditions, use the [`assert`][1] or [`error`][2] functions. +# +# [1] https://www.mathworks.com/help/matlab/ref/assert.html +# [2] https://www.mathworks.com/help/matlab/ref/error.html + +# The `test` runs the MATLAB and Simulink tests in your project. It calls the [`runtests`][3] function +# to run the tests and then the [`assertSuccess`][4] method to fail the job if any of the tests fail. +# +test: + script: matlab -batch "results = runtests('IncludeSubfolders',true), assertSuccess(results);" + +# By default, the job includes any files in your [MATLAB Project][5] that have a `Test` label. If your repository +# does not have a MATLAB project, then the job includes all tests in the root of your repository or in any of +# its subfolders. +# +# [3] https://www.mathworks.com/help/matlab/ref/runtests.html +# [4] https://www.mathworks.com/help/matlab/ref/matlab.unittest.testresult.assertsuccess.html +# [5] https://www.mathworks.com/help/matlab/projects.html + +# The `test_artifacts_job` runs your tests and additionally generates test and coverage artifacts. +# It uses the plugin classes in the [`matlab.unittest.plugins`][6] package to generate a JUnit test results +# report and a Cobertura code coverage report. Like the `run_tests` job, this job runs all the tests in your +# project and fails the build if any of the tests fail. +# +test_artifacts_job: + script: | + matlab -batch " + import matlab.unittest.TestRunner + import matlab.unittest.Verbosity + import matlab.unittest.plugins.CodeCoveragePlugin + import matlab.unittest.plugins.XMLPlugin + import matlab.unittest.plugins.codecoverage.CoberturaFormat + + suite = testsuite(pwd,'IncludeSubfolders',true); + + [~,~] = mkdir('artifacts'); + + runner = TestRunner.withTextOutput('OutputDetail',Verbosity.Detailed); + runner.addPlugin(XMLPlugin.producingJUnitFormat('artifacts/results.xml')) + runner.addPlugin(CodeCoveragePlugin.forFolder(pwd,'IncludingSubfolders',true, ... + 'Producing',CoberturaFormat('artifacts/cobertura.xml'))) + + results = runner.run(suite) + assertSuccess(results);" + + artifacts: + reports: + junit: "./artifacts/results.xml" + cobertura: "./artifacts/cobertura.xml" + paths: + - "./artifacts" + +# You can modify the contents of the `test_artifacts_job` depending on your goals. For more +# information on how to customize the test runner and generate various test and coverage artifacts, +# see [Generate Artifacts Using MATLAB Unit Test Plugins][7]. +# +# [6] https://www.mathworks.com/help/matlab/ref/matlab.unittest.plugins-package.html +# [7] https://www.mathworks.com/help/matlab/matlab_prog/generate-artifacts-using-matlab-unit-test-plugins.html diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index 6ed5e05ed4c..191d5b6b11c 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -13,7 +13,7 @@ variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" # Pip's cache doesn't store the python packages -# https://pip.pypa.io/en/stable/reference/pip_install/#caching +# https://pip.pypa.io/en/stable/topics/caching/ # # If you want to also cache the installed packages, you have to install # them in a virtualenv and cache it as well. diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index bd8ba71effe..b6e811aa84f 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -3,19 +3,36 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: API-Fuzzing.latest.gitlab-ci.yml +# +# You also need to add a `fuzz` stage to your `stages:` configuration. A sample configuration for API Fuzzing: +# +# stages: +# - build +# - test +# - deploy +# - fuzz + # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ # -# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# Configure API Fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: - FUZZAPI_VERSION: "1" + # Setting this variable affects all Security templates + # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + # + FUZZAPI_VERSION: "1" + FUZZAPI_IMAGE_SUFFIX: "" FUZZAPI_IMAGE: api-fuzzing apifuzzer_fuzz: stage: fuzz - image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION + image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION$FUZZAPI_IMAGE_SUFFIX allow_failure: true rules: - if: $API_FUZZING_DISABLED @@ -23,6 +40,10 @@ apifuzzer_fuzz: - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" + variables: + FUZZAPI_IMAGE_SUFFIX: "-fips" - if: $CI_COMMIT_BRANCH script: - /peach/analyzer-fuzz-api diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 65a2b20d5c0..66db311f897 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -25,7 +25,7 @@ variables: CS_ANALYZER_IMAGE: registry.gitlab.com/security-products/container-scanning:4 container_scanning: - image: "$CS_ANALYZER_IMAGE" + image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" stage: test variables: # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your @@ -47,4 +47,10 @@ container_scanning: - if: $CONTAINER_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml index 0e0afa489a3..b491b3e3c0c 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -27,11 +27,12 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # DAST_API_VERSION: "1" + DAST_API_IMAGE_SUFFIX: "" DAST_API_IMAGE: api-fuzzing dast_api: stage: dast - image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION + image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION$DAST_API_IMAGE_SUFFIX allow_failure: true rules: - if: $DAST_API_DISABLED @@ -39,6 +40,10 @@ dast_api: - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_API_IMAGE_SUFFIX: "-fips" - if: $CI_COMMIT_BRANCH script: - /peach/analyzer-dast-api @@ -50,3 +55,5 @@ dast_api: - gl-*.log reports: dast: gl-dast-api-report.json + +# end diff --git a/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml b/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml new file mode 100644 index 00000000000..8a0913e8f66 --- /dev/null +++ b/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml @@ -0,0 +1,27 @@ +# Shopify Theme Kit is a CLI tool for Shopify Themes: https://shopify.github.io/themekit/ +# See the full usage of this template described in: https://medium.com/@gogl.alex/how-to-deploy-shopify-themes-automatically-1ac17ee1229c + +image: python:2 + +stages: + - deploy:staging + - deploy:production + +staging: + image: python:2 + stage: deploy:staging + script: + - curl -s https://shopify.github.io/themekit/scripts/install.py | python + - theme deploy --env=staging + only: + variables: + - $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH + +production: + image: python:2 + stage: deploy:production + script: + - curl -s https://shopify.github.io/themekit/scripts/install.py | python + - theme deploy --env=production --allow-live + only: + - tags diff --git a/lib/gitlab/ci/templates/liquibase.gitlab-ci.yml b/lib/gitlab/ci/templates/liquibase.gitlab-ci.yml new file mode 100644 index 00000000000..18d59035b78 --- /dev/null +++ b/lib/gitlab/ci/templates/liquibase.gitlab-ci.yml @@ -0,0 +1,149 @@ +# This file is a template, and might need editing before it works on your project. +# Here is a live project example that is using this template: +# https://gitlab.com/szandany/h2 + +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/liquibase.gitlab-ci.yml + +# This template must be configured with CI/CD variables before it will work. +# See https://www.liquibase.com/blog/secure-database-developer-flow-using-gitlab-pipelines +# to learn how to configure the Liquibase template by using variables. +# Be sure to add the variables before running pipelines with this template. +# You may not want to run all the jobs in this template. You can comment out or delete the jobs you don't wish to use. + +# List of stages for jobs and their order of execution. +stages: + - build + - test + - deploy + - compare + + +# Helper functions to determine if the database is ready for deployments (function isUpToDate) or rollbacks (function isRollback) when tag is applied. +.functions: &functions | + function isUpToDate(){ + status=$(liquibase status --verbose) + if [[ $status == *'is up to date'* ]]; then + echo "database is already up to date" & exit 0 + fi; + } + + function isRollback(){ + if [ -z "$TAG" ]; then + echo "No TAG provided, running any pending changes" + elif [[ "$(liquibase rollbackSQL $TAG)" ]]; then + liquibase --logLevel=info --logFile=${CI_JOB_NAME}_${CI_PIPELINE_ID}.log rollback $TAG && exit 0 + else exit 0 + fi; + } + + +# This is a series of Liquibase commands that can be run while doing database migrations from Liquibase docs at https://docs.liquibase.com/commands/home.html +.liquibase_job: + image: liquibase/liquibase:latest # Using the Liquibase Docker Image at - https://hub.docker.com/r/liquibase/liquibase + before_script: + - liquibase --version + - *functions + - isRollback + - isUpToDate + - liquibase checks run + - liquibase update + - liquibase rollbackOneUpdate --force # This is a Pro command. Try Pro free trial here - https://liquibase.org/try-liquibase-pro-free + - liquibase tag $CI_PIPELINE_ID + - liquibase --logFile=${CI_JOB_NAME}_${CI_PIPELINE_ID}.log --logLevel=info update + - liquibase history + artifacts: + paths: + - ${CI_JOB_NAME}_${CI_PIPELINE_ID}.log + expire_in: 1 week + + +# This job runs in the build stage, which runs first. +build-job: + extends: .liquibase_job + stage: build + environment: + name: DEV + script: + - echo "This job tested successfully with liquibase in DEV environment" + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + + +# This job runs in the test stage. It only starts when the job in the build stage completes successfully. +test-job: + extends: .liquibase_job + stage: test + environment: + name: TEST + script: + - echo "This job testsed successfully with liquibase in TEST environment" + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + + +# This job runs in the deploy stage. It only starts when the jobs in the test stage completes successfully. +deploy-prod: + extends: .liquibase_job + stage: deploy + environment: + name: PROD + script: + - echo "This job deployed successfully Liquibase in a production environment from the $CI_COMMIT_BRANCH branch." + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + + +# This job compares dev database with test database to detect any drifts in the pipeline. Learn more about comparing database with Liquibase here https://docs.liquibase.com/commands/diff.html +DEV->TEST: + image: liquibase/liquibase:latest # Using the Liquibase Docker Image + stage: compare + environment: + name: TEST + script: + - echo "Comparing databases DEV --> TEST" + - liquibase diff + - liquibase --outputFile=diff_between_DEV_TEST.json diff --format=json + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + artifacts: + paths: + - diff_between_DEV_TEST.json + expire_in: 1 week + + +# This job compares test database with prod database to detect any drifts in the pipeline. +TEST->PROD: + image: liquibase/liquibase:latest # Using the Liquibase Docker Image + stage: compare + environment: + name: PROD + script: + - echo "Comparing databases TEST --> PROD" + - liquibase diff + - liquibase --outputFile=diff_between_TEST_PROD.json diff --format=json + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + artifacts: + paths: + - diff_between_TEST_PROD.json + expire_in: 1 week + + +# This job creates a snapshot of prod database. You can use the snapshot file to run comparisons with the production database to investigate for any potential issues. https://www.liquibase.com/devsecops +snapshot PROD: + image: liquibase/liquibase:latest # Using the Liquibase Docker Image + stage: .post + environment: + name: PROD + script: + - echo "Snapshotting database PROD" + - liquibase --outputFile=snapshot_PROD_${CI_PIPELINE_ID}.json snapshot --snapshotFormat=json --log-level debug + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + artifacts: + paths: + - snapshot_PROD_${CI_PIPELINE_ID}.json + expire_in: 1 week diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index bfcf67693e7..bcb1fe83ea2 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -10,7 +10,7 @@ module Gitlab @pipeline = pipeline @instance_variables_builder = Builder::Instance.new @project_variables_builder = Builder::Project.new(project) - @group_variables_builder = Builder::Group.new(project.group) + @group_variables_builder = Builder::Group.new(project&.group) end def scoped_variables(job, environment:, dependencies:) @@ -24,11 +24,25 @@ module Gitlab variables.concat(user_variables(job.user)) variables.concat(job.dependency_variables) if dependencies variables.concat(secret_instance_variables) - variables.concat(secret_group_variables(environment: environment, ref: job.git_ref)) - variables.concat(secret_project_variables(environment: environment, ref: job.git_ref)) + variables.concat(secret_group_variables(environment: environment)) + variables.concat(secret_project_variables(environment: environment)) variables.concat(job.trigger_request.user_variables) if job.trigger_request variables.concat(pipeline.variables) - variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule + variables.concat(pipeline_schedule_variables) + end + end + + def config_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + break variables unless project + + variables.concat(project.predefined_variables) + variables.concat(pipeline.predefined_variables) + variables.concat(secret_instance_variables) + variables.concat(secret_group_variables(environment: nil)) + variables.concat(secret_project_variables(environment: nil)) + variables.concat(pipeline.variables) + variables.concat(pipeline_schedule_variables) end end @@ -75,21 +89,21 @@ module Gitlab end end - def secret_group_variables(environment:, ref:) - if memoize_secret_variables? - memoized_secret_group_variables(environment: environment) - else - return [] unless project.group - - project.group.ci_variables_for(ref, project, environment: environment) + def secret_group_variables(environment:) + strong_memoize_with(:secret_group_variables, environment) do + group_variables_builder + .secret_variables( + environment: environment, + protected_ref: protected_ref?) end end - def secret_project_variables(environment:, ref:) - if memoize_secret_variables? - memoized_secret_project_variables(environment: environment) - else - project.ci_variables_for(ref: ref, environment: environment) + def secret_project_variables(environment:) + strong_memoize_with(:secret_project_variables, environment) do + project_variables_builder + .secret_variables( + environment: environment, + protected_ref: protected_ref?) end end @@ -120,21 +134,15 @@ module Gitlab end end - def memoized_secret_project_variables(environment:) - strong_memoize_with(:secret_project_variables, environment) do - project_variables_builder - .secret_variables( - environment: environment, - protected_ref: protected_ref?) - end - end + def pipeline_schedule_variables + strong_memoize(:pipeline_schedule_variables) do + variables = if pipeline.pipeline_schedule + pipeline.pipeline_schedule.job_variables + else + [] + end - def memoized_secret_group_variables(environment:) - strong_memoize_with(:secret_group_variables, environment) do - group_variables_builder - .secret_variables( - environment: environment, - protected_ref: protected_ref?) + Gitlab::Ci::Variables::Collection.new(variables) end end @@ -150,14 +158,6 @@ module Gitlab end end - def memoize_secret_variables? - strong_memoize(:memoize_secret_variables) do - ::Feature.enabled?(:ci_variables_builder_memoize_secret_variables, - project, - default_enabled: :yaml) - end - end - def strong_memoize_with(name, *args) container = strong_memoize(name) { {} } diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 0d4b913b7a0..22a4ba8ac7a 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -22,7 +22,7 @@ module Gitlab 'frame_src' => ContentSecurityPolicy::Directives.frame_src, 'img_src' => "'self' data: blob: http: https:", 'manifest_src' => "'self'", - 'media_src' => "'self'", + 'media_src' => "'self' data:", 'script_src' => ContentSecurityPolicy::Directives.script_src, 'style_src' => "'self' 'unsafe-inline'", 'worker_src' => "#{Gitlab::Utils.append_path(Gitlab.config.gitlab.url, 'assets/')} blob: data:", @@ -37,13 +37,13 @@ module Gitlab allow_webpack_dev_server(directives) allow_letter_opener(directives) allow_snowplow_micro(directives) if Gitlab::Tracking.snowplow_micro_enabled? - allow_customersdot(directives) if ENV['CUSTOMER_PORTAL_URL'].present? end allow_websocket_connections(directives) allow_cdn(directives, Settings.gitlab.cdn_host) if Settings.gitlab.cdn_host.present? allow_sentry(directives) if Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn allow_framed_gitlab_paths(directives) + allow_customersdot(directives) if ENV['CUSTOMER_PORTAL_URL'].present? # The follow section contains workarounds to patch Safari's lack of support for CSP Level 3 # See https://gitlab.com/gitlab-org/gitlab/-/issues/343579 diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index a4508bc93c5..0e6841e10a7 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -12,6 +12,16 @@ module Gitlab Gitlab::UrlBuilder.build(deployment.deployable) end + commit_url = + if (commit = deployment.commit) + Gitlab::UrlBuilder.build(commit) + end + + user_url = + if deployment.deployed_by + Gitlab::UrlBuilder.build(deployment.deployed_by) + end + { object_kind: 'deployment', status: deployment.status, @@ -22,10 +32,10 @@ module Gitlab environment: deployment.environment.name, project: deployment.project.hook_attrs, short_sha: deployment.short_sha, - user: deployment.deployed_by.hook_attrs, - user_url: Gitlab::UrlBuilder.build(deployment.deployed_by), - commit_url: Gitlab::UrlBuilder.build(deployment.commit), - commit_title: deployment.commit.title, + user: deployment.deployed_by&.hook_attrs, + user_url: user_url, + commit_url: commit_url, + commit_title: deployment.commit_title, ref: deployment.ref } end diff --git a/lib/gitlab/data_builder/note.rb b/lib/gitlab/data_builder/note.rb index 73518d36d43..dec583f5a42 100644 --- a/lib/gitlab/data_builder/note.rb +++ b/lib/gitlab/data_builder/note.rb @@ -43,10 +43,9 @@ module Gitlab if note.for_commit? data[:commit] = build_data_for_commit(project, user, note) elsif note.for_issue? - data[:issue] = note.noteable.hook_attrs - data[:issue][:labels] = note.noteable.labels_hook_attrs + data[:issue] = Gitlab::HookData::IssueBuilder.new(note.noteable).build elsif note.for_merge_request? - data[:merge_request] = note.noteable.hook_attrs + data[:merge_request] = Gitlab::HookData::MergeRequestBuilder.new(note.noteable).build elsif note.for_snippet? data[:snippet] = note.noteable.hook_attrs end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 1b16873f737..1895f0fab32 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -161,24 +161,6 @@ module Gitlab end end - def self.nulls_order(field, direction = :asc, nulls_order = :nulls_last) - raise ArgumentError unless [:nulls_last, :nulls_first].include?(nulls_order) - raise ArgumentError unless [:asc, :desc].include?(direction) - - case nulls_order - when :nulls_last then nulls_last_order(field, direction) - when :nulls_first then nulls_first_order(field, direction) - end - end - - def self.nulls_last_order(field, direction = 'ASC') - Arel.sql("#{field} #{direction} NULLS LAST") - end - - def self.nulls_first_order(field, direction = 'ASC') - Arel.sql("#{field} #{direction} NULLS FIRST") - end - def self.random "RANDOM()" end @@ -228,7 +210,7 @@ module Gitlab end def self.db_config_names - ::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name) + ::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name) - ['geo'] end # This returns all matching schemas that a given connection can use @@ -236,13 +218,16 @@ module Gitlab # This does not look at literal connection names, but rather compares # models that are holders for a given db_config_name def self.gitlab_schemas_for_connection(connection) - connection_name = self.db_config_name(connection) - primary_model = self.database_base_models.fetch(connection_name) - - self.schemas_to_base_models - .select { |_, models| models.include?(primary_model) } - .keys - .map!(&:to_sym) + db_name = self.db_config_name(connection) + primary_model = self.database_base_models.fetch(db_name.to_sym) + + self.schemas_to_base_models.select do |_, child_models| + child_models.any? do |child_model| + child_model == primary_model || \ + # The model might indicate a child connection, ensure that this is enclosed in a `db_config` + self.database_base_models[self.db_config_share_with(child_model.connection_db_config)] == primary_model + end + end.keys.map!(&:to_sym) end def self.db_config_for_connection(connection) @@ -271,6 +256,17 @@ module Gitlab db_config&.name || 'unknown' end + # Currently the database configuration can only be shared with `main:` + # If the `database_tasks: false` is being used + # This is to be refined: https://gitlab.com/gitlab-org/gitlab/-/issues/356580 + def self.db_config_share_with(db_config) + if db_config.database_tasks? + nil # no sharing + else + 'main' # share with `main:` + end + end + def self.read_only? false end diff --git a/lib/gitlab/database/background_migration/batch_metrics.rb b/lib/gitlab/database/background_migration/batch_metrics.rb index 3e6d7ac3c9f..14fe0c14c24 100644 --- a/lib/gitlab/database/background_migration/batch_metrics.rb +++ b/lib/gitlab/database/background_migration/batch_metrics.rb @@ -5,17 +5,24 @@ module Gitlab module BackgroundMigration class BatchMetrics attr_reader :timings + attr_reader :affected_rows def initialize @timings = {} + @affected_rows = {} end - def time_operation(label) + def time_operation(label, &blk) + instrument_operation(label, instrument_affected_rows: false, &blk) + end + + def instrument_operation(label, instrument_affected_rows: true) start_time = monotonic_time - yield + count = yield timings_for_label(label) << monotonic_time - start_time + affected_rows_for_label(label) << count if instrument_affected_rows && count.is_a?(Integer) end private @@ -24,6 +31,10 @@ module Gitlab timings[label] ||= [] end + def affected_rows_for_label(label) + affected_rows[label] ||= [] + end + def monotonic_time Gitlab::Metrics::System.monotonic_time end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index f3160679d64..ebc3ee240bd 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -25,6 +25,7 @@ module Gitlab scope :except_succeeded, -> { without_status(:succeeded) } scope :successful_in_execution_order, -> { where.not(finished_at: nil).with_status(:succeeded).order(:finished_at) } scope :with_preloads, -> { preload(:batched_migration) } + scope :created_since, ->(date_time) { where('created_at >= ?', date_time) } state_machine :status, initial: :pending do state :pending, value: 0 @@ -62,7 +63,13 @@ module Gitlab job.split_and_retry! if job.can_split?(exception) rescue SplitAndRetryError => error - Gitlab::AppLogger.error(message: error.message, batched_job_id: job.id) + Gitlab::AppLogger.error( + message: error.message, + batched_job_id: job.id, + batched_migration_id: job.batched_migration.id, + job_class_name: job.migration_job_class_name, + job_arguments: job.migration_job_arguments + ) end after_transition do |job, transition| @@ -72,13 +79,23 @@ module Gitlab job.batched_job_transition_logs.create(previous_status: transition.from, next_status: transition.to, exception_class: exception&.class, exception_message: exception&.message) - Gitlab::ErrorTracking.track_exception(exception, batched_job_id: job.id) if exception - - Gitlab::AppLogger.info(message: 'BatchedJob transition', batched_job_id: job.id, previous_state: transition.from_name, new_state: transition.to_name) + Gitlab::ErrorTracking.track_exception(exception, batched_job_id: job.id, job_class_name: job.migration_job_class_name, job_arguments: job.migration_job_arguments) if exception + + Gitlab::AppLogger.info( + message: 'BatchedJob transition', + batched_job_id: job.id, + previous_state: transition.from_name, + new_state: transition.to_name, + batched_migration_id: job.batched_migration.id, + job_class_name: job.migration_job_class_name, + job_arguments: job.migration_job_arguments, + exception_class: exception&.class, + exception_message: exception&.message + ) end end - delegate :job_class, :table_name, :column_name, :job_arguments, + delegate :job_class, :table_name, :column_name, :job_arguments, :job_class_name, to: :batched_migration, prefix: :migration attribute :pause_ms, :integer, default: 100 diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 65c15795de6..d94bf060d05 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -6,6 +6,8 @@ module Gitlab class BatchedMigration < SharedModel JOB_CLASS_MODULE = 'Gitlab::BackgroundMigration' BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies" + MAXIMUM_FAILED_RATIO = 0.5 + MINIMUM_JOBS = 50 self.table_name = :batched_background_migrations @@ -21,28 +23,60 @@ module Gitlab validate :validate_batched_jobs_status, if: -> { status_changed? && finished? } scope :queue_order, -> { order(id: :asc) } - scope :queued, -> { where(status: [:active, :paused]) } + scope :queued, -> { with_statuses(:active, :paused) } + + # on_hold_until is a temporary runtime status which puts execution "on hold" + scope :executable, -> { with_status(:active).where('on_hold_until IS NULL OR on_hold_until < NOW()') } + scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do where(job_class_name: job_class_name, table_name: table_name, column_name: column_name) .where("job_arguments = ?", job_arguments.to_json) # rubocop:disable Rails/WhereEquals end - enum status: { - paused: 0, - active: 1, - finished: 3, - failed: 4, - finalizing: 5 - } + state_machine :status, initial: :paused do + state :paused, value: 0 + state :active, value: 1 + state :finished, value: 3 + state :failed, value: 4 + state :finalizing, value: 5 + + event :pause do + transition any => :paused + end + + event :execute do + transition any => :active + end + + event :finish do + transition any => :finished + end + + event :failure do + transition any => :failed + end + + event :finalize do + transition any => :finalizing + end + + before_transition any => :active do |migration| + migration.started_at = Time.current if migration.respond_to?(:started_at) + end + end attribute :pause_ms, :integer, default: 100 + def self.valid_status + state_machine.states.map(&:name) + end + def self.find_for_configuration(job_class_name, table_name, column_name, job_arguments) for_configuration(job_class_name, table_name, column_name, job_arguments).first end def self.active_migration - active.queue_order.first + executable.queue_order.first end def self.successful_rows_counts(migrations) @@ -74,11 +108,23 @@ module Gitlab batched_jobs.with_status(:failed).each_batch(of: 100) do |batch| self.class.transaction do batch.lock.each(&:split_and_retry!) - self.active! + self.execute! end end - self.active! + self.execute! + end + + def should_stop? + return unless started_at + + total_jobs = batched_jobs.created_since(started_at).count + + return if total_jobs < MINIMUM_JOBS + + failed_jobs = batched_jobs.with_status(:failed).created_since(started_at).count + + failed_jobs.fdiv(total_jobs) > MAXIMUM_FAILED_RATIO end def next_min_value @@ -136,6 +182,10 @@ module Gitlab BatchOptimizer.new(self).optimize! end + def hold!(until_time: 10.minutes.from_now) + update!(on_hold_until: until_time) + end + private def validate_batched_jobs_status diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 06cd40f1e06..59ff9a9744f 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -6,13 +6,13 @@ module Gitlab class BatchedMigrationRunner FailedToFinalize = Class.new(RuntimeError) - def self.finalize(job_class_name, table_name, column_name, job_arguments, connection: ApplicationRecord.connection) + def self.finalize(job_class_name, table_name, column_name, job_arguments, connection:) new(connection: connection).finalize(job_class_name, table_name, column_name, job_arguments) end - def initialize(migration_wrapper = BatchedMigrationWrapper.new, connection: ApplicationRecord.connection) - @migration_wrapper = migration_wrapper + def initialize(connection:, migration_wrapper: BatchedMigrationWrapper.new(connection: connection)) @connection = connection + @migration_wrapper = migration_wrapper end # Runs the next batched_job for a batched_background_migration. @@ -30,6 +30,7 @@ module Gitlab migration_wrapper.perform(next_batched_job) active_migration.optimize! + active_migration.failure! if next_batched_job.failed? && active_migration.should_stop? else finish_active_migration(active_migration) end @@ -67,7 +68,7 @@ module Gitlab elsif migration.finished? Gitlab::AppLogger.warn "Batched background migration for the given configuration is already finished: #{configuration}" else - migration.finalizing! + migration.finalize! migration.batched_jobs.with_status(:pending).each { |job| migration_wrapper.perform(job) } run_migration_while(migration, :finalizing) @@ -78,7 +79,7 @@ module Gitlab private - attr_reader :migration_wrapper, :connection + attr_reader :connection, :migration_wrapper def find_or_create_next_batched_job(active_migration) if next_batch_range = find_next_batch_range(active_migration) @@ -118,14 +119,14 @@ module Gitlab return if active_migration.batched_jobs.active.exists? if active_migration.batched_jobs.with_status(:failed).exists? - active_migration.failed! + active_migration.failure! else - active_migration.finished! + active_migration.finish! end end def run_migration_while(migration, status) - while migration.status == status.to_s + while migration.status_name == status run_migration_job(migration) migration.reload_last_job diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb index 057f856d859..ec68f401ca2 100644 --- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -4,10 +4,9 @@ module Gitlab module Database module BackgroundMigration class BatchedMigrationWrapper - extend Gitlab::Utils::StrongMemoize - - def initialize(connection: ApplicationRecord.connection) + def initialize(connection:, metrics: PrometheusMetrics.new) @connection = connection + @metrics = metrics end # Wraps the execution of a batched_background_migration. @@ -28,12 +27,12 @@ module Gitlab raise ensure - track_prometheus_metrics(batch_tracking_record) + metrics.track(batch_tracking_record) end private - attr_reader :connection + attr_reader :connection, :metrics def start_tracking_execution(tracking_record) tracking_record.run! @@ -63,80 +62,6 @@ module Gitlab job_class.new end end - - def track_prometheus_metrics(tracking_record) - migration = tracking_record.batched_migration - base_labels = migration.prometheus_labels - - metric_for(:gauge_batch_size).set(base_labels, tracking_record.batch_size) - metric_for(:gauge_sub_batch_size).set(base_labels, tracking_record.sub_batch_size) - metric_for(:gauge_interval).set(base_labels, tracking_record.batched_migration.interval) - metric_for(:gauge_job_duration).set(base_labels, (tracking_record.finished_at - tracking_record.started_at).to_i) - metric_for(:counter_updated_tuples).increment(base_labels, tracking_record.batch_size) - metric_for(:gauge_migrated_tuples).set(base_labels, tracking_record.batched_migration.migrated_tuple_count) - metric_for(:gauge_total_tuple_count).set(base_labels, tracking_record.batched_migration.total_tuple_count) - metric_for(:gauge_last_update_time).set(base_labels, Time.current.to_i) - - if metrics = tracking_record.metrics - metrics['timings']&.each do |key, timings| - summary = metric_for(:histogram_timings) - labels = base_labels.merge(operation: key) - - timings.each do |timing| - summary.observe(labels, timing) - end - end - end - end - - def metric_for(name) - self.class.metrics[name] - end - - def self.metrics - strong_memoize(:metrics) do - { - gauge_batch_size: Gitlab::Metrics.gauge( - :batched_migration_job_batch_size, - 'Batch size for a batched migration job' - ), - gauge_sub_batch_size: Gitlab::Metrics.gauge( - :batched_migration_job_sub_batch_size, - 'Sub-batch size for a batched migration job' - ), - gauge_interval: Gitlab::Metrics.gauge( - :batched_migration_job_interval_seconds, - 'Interval for a batched migration job' - ), - gauge_job_duration: Gitlab::Metrics.gauge( - :batched_migration_job_duration_seconds, - 'Duration for a batched migration job' - ), - counter_updated_tuples: Gitlab::Metrics.counter( - :batched_migration_job_updated_tuples_total, - 'Number of tuples updated by batched migration job' - ), - gauge_migrated_tuples: Gitlab::Metrics.gauge( - :batched_migration_migrated_tuples_total, - 'Total number of tuples migrated by a batched migration' - ), - histogram_timings: Gitlab::Metrics.histogram( - :batched_migration_job_query_duration_seconds, - 'Query timings for a batched migration job', - {}, - [0.1, 0.25, 0.5, 1, 5].freeze - ), - gauge_total_tuple_count: Gitlab::Metrics.gauge( - :batched_migration_total_tuple_count, - 'Total tuple count the migration needs to touch' - ), - gauge_last_update_time: Gitlab::Metrics.gauge( - :batched_migration_last_update_time_seconds, - 'Unix epoch time in seconds' - ) - } - end - end end end end diff --git a/lib/gitlab/database/background_migration/prometheus_metrics.rb b/lib/gitlab/database/background_migration/prometheus_metrics.rb new file mode 100644 index 00000000000..ce1da4c59eb --- /dev/null +++ b/lib/gitlab/database/background_migration/prometheus_metrics.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class PrometheusMetrics + extend Gitlab::Utils::StrongMemoize + + QUERY_TIMING_BUCKETS = [0.1, 0.25, 0.5, 1, 5].freeze + + def track(job_record) + migration_record = job_record.batched_migration + base_labels = migration_record.prometheus_labels + + metric_for(:gauge_batch_size).set(base_labels, job_record.batch_size) + metric_for(:gauge_sub_batch_size).set(base_labels, job_record.sub_batch_size) + metric_for(:gauge_interval).set(base_labels, job_record.batched_migration.interval) + metric_for(:gauge_job_duration).set(base_labels, (job_record.finished_at - job_record.started_at).to_i) + metric_for(:counter_updated_tuples).increment(base_labels, job_record.batch_size) + metric_for(:gauge_migrated_tuples).set(base_labels, migration_record.migrated_tuple_count) + metric_for(:gauge_total_tuple_count).set(base_labels, migration_record.total_tuple_count) + metric_for(:gauge_last_update_time).set(base_labels, Time.current.to_i) + + track_timing_metrics(base_labels, job_record.metrics) + end + + def self.metrics + strong_memoize(:metrics) do + { + gauge_batch_size: Gitlab::Metrics.gauge( + :batched_migration_job_batch_size, + 'Batch size for a batched migration job' + ), + gauge_sub_batch_size: Gitlab::Metrics.gauge( + :batched_migration_job_sub_batch_size, + 'Sub-batch size for a batched migration job' + ), + gauge_interval: Gitlab::Metrics.gauge( + :batched_migration_job_interval_seconds, + 'Interval for a batched migration job' + ), + gauge_job_duration: Gitlab::Metrics.gauge( + :batched_migration_job_duration_seconds, + 'Duration for a batched migration job' + ), + counter_updated_tuples: Gitlab::Metrics.counter( + :batched_migration_job_updated_tuples_total, + 'Number of tuples updated by batched migration job' + ), + gauge_migrated_tuples: Gitlab::Metrics.gauge( + :batched_migration_migrated_tuples_total, + 'Total number of tuples migrated by a batched migration' + ), + histogram_timings: Gitlab::Metrics.histogram( + :batched_migration_job_query_duration_seconds, + 'Query timings for a batched migration job', + {}, + QUERY_TIMING_BUCKETS + ), + gauge_total_tuple_count: Gitlab::Metrics.gauge( + :batched_migration_total_tuple_count, + 'Total tuple count the migration needs to touch' + ), + gauge_last_update_time: Gitlab::Metrics.gauge( + :batched_migration_last_update_time_seconds, + 'Unix epoch time in seconds' + ) + } + end + end + + private + + def track_timing_metrics(base_labels, metrics) + return unless metrics && metrics['timings'] + + metrics['timings'].each do |key, timings| + summary = metric_for(:histogram_timings) + labels = base_labels.merge(operation: key) + + timings.each do |timing| + summary.observe(labels, timing) + end + end + end + + def metric_for(name) + self.class.metrics[name] + end + end + end + end +end diff --git a/lib/gitlab/database/consistency_checker.rb b/lib/gitlab/database/consistency_checker.rb new file mode 100644 index 00000000000..e398fef744c --- /dev/null +++ b/lib/gitlab/database/consistency_checker.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class ConsistencyChecker + BATCH_SIZE = 1000 + MAX_BATCHES = 25 + MAX_RUNTIME = 30.seconds # must be less than the scheduling frequency of the ConsistencyCheck jobs + + delegate :monotonic_time, to: :'Gitlab::Metrics::System' + + def initialize(source_model:, target_model:, source_columns:, target_columns:) + @source_model = source_model + @target_model = target_model + @source_columns = source_columns + @target_columns = target_columns + @source_sort_column = source_columns.first + @target_sort_column = target_columns.first + @result = { matches: 0, mismatches: 0, batches: 0, mismatches_details: [] } + end + + # rubocop:disable Metrics/AbcSize + def execute(start_id:) + current_start_id = start_id + + return build_result(next_start_id: nil) if max_id.nil? + return build_result(next_start_id: min_id) if current_start_id > max_id + + @start_time = monotonic_time + + MAX_BATCHES.times do + if (current_start_id <= max_id) && !over_time_limit? + ids_range = current_start_id...(current_start_id + BATCH_SIZE) + # rubocop: disable CodeReuse/ActiveRecord + source_data = source_model.where(source_sort_column => ids_range) + .order(source_sort_column => :asc).pluck(*source_columns) + target_data = target_model.where(target_sort_column => ids_range) + .order(target_sort_column => :asc).pluck(*target_columns) + # rubocop: enable CodeReuse/ActiveRecord + + current_start_id += BATCH_SIZE + result[:matches] += append_mismatches_details(source_data, target_data) + result[:batches] += 1 + else + break + end + end + + result[:mismatches] = result[:mismatches_details].length + metrics_counter.increment({ source_table: source_model.table_name, result: "match" }, result[:matches]) + metrics_counter.increment({ source_table: source_model.table_name, result: "mismatch" }, result[:mismatches]) + + build_result(next_start_id: current_start_id > max_id ? min_id : current_start_id) + end + # rubocop:enable Metrics/AbcSize + + private + + attr_reader :source_model, :target_model, :source_columns, :target_columns, + :source_sort_column, :target_sort_column, :start_time, :result + + def build_result(next_start_id:) + { next_start_id: next_start_id }.merge(result) + end + + def over_time_limit? + (monotonic_time - start_time) >= MAX_RUNTIME + end + + # This where comparing the items happen, and building the diff log + # It returns the number of matching elements + def append_mismatches_details(source_data, target_data) + # Mapping difference the sort key to the item values + # source - target + source_diff_hash = (source_data - target_data).index_by { |item| item.shift } + # target - source + target_diff_hash = (target_data - source_data).index_by { |item| item.shift } + + matches = source_data.length - source_diff_hash.length + + # Items that exist in the first table + Different items + source_diff_hash.each do |id, values| + result[:mismatches_details] << { + id: id, + source_table: values, + target_table: target_diff_hash[id] + } + end + + # Only the items that exist in the target table + target_diff_hash.each do |id, values| + next if source_diff_hash[id] # It's already added + + result[:mismatches_details] << { + id: id, + source_table: source_diff_hash[id], + target_table: values + } + end + + matches + end + + # rubocop: disable CodeReuse/ActiveRecord + def min_id + @min_id ||= source_model.minimum(source_sort_column) + end + + def max_id + @max_id ||= source_model.maximum(source_sort_column) + end + # rubocop: enable CodeReuse/ActiveRecord + + def metrics_counter + @metrics_counter ||= Gitlab::Metrics.counter( + :consistency_checks, + "Consistency Check Results" + ) + end + end + end +end diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb index cccd4b48723..0d876f5124f 100644 --- a/lib/gitlab/database/each_database.rb +++ b/lib/gitlab/database/each_database.rb @@ -4,11 +4,13 @@ module Gitlab module Database module EachDatabase class << self - def each_database_connection(only: nil) + def each_database_connection(only: nil, include_shared: true) selected_names = Array.wrap(only) base_models = select_base_models(selected_names) base_models.each_pair do |connection_name, model| + next if !include_shared && Gitlab::Database.db_config_share_with(model.connection_db_config) + connection = model.connection with_shared_connection(connection, connection_name) do diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index dcd78bfd84f..ae0ea919b62 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -42,11 +42,11 @@ audit_events: :gitlab_main authentication_events: :gitlab_main award_emoji: :gitlab_main aws_roles: :gitlab_main -background_migration_jobs: :gitlab_main +background_migration_jobs: :gitlab_shared badges: :gitlab_main banned_users: :gitlab_main -batched_background_migration_jobs: :gitlab_main -batched_background_migrations: :gitlab_main +batched_background_migration_jobs: :gitlab_shared +batched_background_migrations: :gitlab_shared board_assignees: :gitlab_main board_group_recent_visits: :gitlab_main board_labels: :gitlab_main @@ -240,6 +240,7 @@ group_deletion_schedules: :gitlab_main group_deploy_keys: :gitlab_main group_deploy_keys_groups: :gitlab_main group_deploy_tokens: :gitlab_main +group_features: :gitlab_main group_group_links: :gitlab_main group_import_states: :gitlab_main group_merge_request_approval_settings: :gitlab_main @@ -393,7 +394,7 @@ postgres_indexes: :gitlab_shared postgres_partitioned_tables: :gitlab_shared postgres_partitions: :gitlab_shared postgres_reindex_actions: :gitlab_shared -postgres_reindex_queued_actions: :gitlab_main +postgres_reindex_queued_actions: :gitlab_shared product_analytics_events_experimental: :gitlab_main programming_languages: :gitlab_main project_access_tokens: :gitlab_main @@ -435,6 +436,7 @@ protected_branches: :gitlab_main protected_branch_merge_access_levels: :gitlab_main protected_branch_push_access_levels: :gitlab_main protected_branch_unprotect_access_levels: :gitlab_main +protected_environment_approval_rules: :gitlab_main protected_environment_deploy_access_levels: :gitlab_main protected_environments: :gitlab_main protected_tag_create_access_levels: :gitlab_main @@ -558,4 +560,4 @@ x509_commit_signatures: :gitlab_main x509_issuers: :gitlab_main zentao_tracker_data: :gitlab_main zoom_meetings: :gitlab_main -batched_background_migration_job_transition_logs: :gitlab_main +batched_background_migration_job_transition_logs: :gitlab_shared diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index 86b3afaa47b..3f03d9e2c12 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -78,15 +78,15 @@ module Gitlab end def primary_model_or_model_if_enabled - if force_no_sharing_primary_model? + if use_dedicated_connection? @model else @primary_model || @model end end - def force_no_sharing_primary_model? - return false unless @primary_model # Doesn't matter since we don't have an overriding primary model + def use_dedicated_connection? + return true unless @primary_model # We can only use dedicated connection, if re-use of connections is disabled return false unless ::Gitlab::SafeRequestStore.active? ::Gitlab::SafeRequestStore.fetch(:force_no_sharing_primary_model) do diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index a91df2eccdd..1be63da8896 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -13,13 +13,6 @@ module Gitlab WriteInsideReadOnlyTransactionError = Class.new(StandardError) READ_ONLY_TRANSACTION_KEY = :load_balacing_read_only_transaction - # The load balancer returned by connection might be different - # between `model.connection.load_balancer` vs `model.load_balancer` - # - # The used `model.connection` is dependent on `use_model_load_balancing`. - # See more in: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73949. - # - # Always use `model.load_balancer` or `model.sticking`. attr_reader :load_balancer # These methods perform writes after which we need to stick to the diff --git a/lib/gitlab/database/load_balancing/setup.rb b/lib/gitlab/database/load_balancing/setup.rb index 6d667e8ecf0..eceea1d8d9c 100644 --- a/lib/gitlab/database/load_balancing/setup.rb +++ b/lib/gitlab/database/load_balancing/setup.rb @@ -17,7 +17,12 @@ module Gitlab configure_connection setup_connection_proxy setup_service_discovery - setup_feature_flag_to_model_load_balancing + + ::Gitlab::Database::LoadBalancing::Logger.debug( + event: :setup, + model: model.name, + start_service_discovery: @start_service_discovery + ) end def configure_connection @@ -45,21 +50,6 @@ module Gitlab setup_class_attribute(:sticking, Sticking.new(load_balancer)) end - # TODO: This is temporary code to gradually redirect traffic to use - # a dedicated DB replicas, or DB primaries (depending on configuration) - # This implements a sticky behavior for the current request if enabled. - # - # This is needed for Phase 3 and Phase 4 of application rollout - # https://gitlab.com/groups/gitlab-org/-/epics/6160#progress - # - # If `GITLAB_USE_MODEL_LOAD_BALANCING` is set, its value is preferred - # Otherwise, a `use_model_load_balancing` FF value is used - def setup_feature_flag_to_model_load_balancing - return if active_record_base? - - @model.singleton_class.prepend(ModelLoadBalancingFeatureFlagMixin) - end - def setup_service_discovery return unless configuration.service_discovery_enabled? @@ -84,31 +74,6 @@ module Gitlab def active_record_base? @model == ActiveRecord::Base end - - module ModelLoadBalancingFeatureFlagMixin - extend ActiveSupport::Concern - - def use_model_load_balancing? - # Cache environment variable and return env variable first if defined - default_use_model_load_balancing_env = Gitlab.dev_or_test_env? || nil - use_model_load_balancing_env = Gitlab::Utils.to_boolean(ENV.fetch('GITLAB_USE_MODEL_LOAD_BALANCING', default_use_model_load_balancing_env)) - - unless use_model_load_balancing_env.nil? - return use_model_load_balancing_env - end - - # Check a feature flag using RequestStore (if active) - return false unless Gitlab::SafeRequestStore.active? - - Gitlab::SafeRequestStore.fetch(:use_model_load_balancing) do - Feature.enabled?(:use_model_load_balancing, default_enabled: :yaml) - end - end - - def connection - use_model_load_balancing? ? super : ApplicationRecord.connection - end - end end end end diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb index b2248b0f4eb..dc695a74a4b 100644 --- a/lib/gitlab/database/migration.rb +++ b/lib/gitlab/database/migration.rb @@ -33,20 +33,33 @@ module Gitlab # We use major version bumps to indicate significant changes and minor version bumps # to indicate backwards-compatible or otherwise minor changes (e.g. a Rails version bump). # However, this hasn't been strictly formalized yet. - MIGRATION_CLASSES = { - 1.0 => Class.new(ActiveRecord::Migration[6.1]) do - include LockRetriesConcern - include Gitlab::Database::MigrationHelpers::V2 + + class V1_0 < ActiveRecord::Migration[6.1] # rubocop:disable Naming/ClassAndModuleCamelCase + include LockRetriesConcern + include Gitlab::Database::MigrationHelpers::V2 + end + + class V2_0 < V1_0 # rubocop:disable Naming/ClassAndModuleCamelCase + include Gitlab::Database::MigrationHelpers::RestrictGitlabSchema + + # When running migrations, the `db:migrate` switches connection of + # ActiveRecord::Base depending where the migration runs. + # This helper class is provided to avoid confusion using `ActiveRecord::Base` + class MigrationRecord < ActiveRecord::Base end - }.freeze + end def self.[](version) - MIGRATION_CLASSES[version] || raise(ArgumentError, "Unknown migration version: #{version}") + version = version.to_s + name = "V#{version.tr('.', '_')}" + raise ArgumentError, "Unknown migration version: #{version}" unless const_defined?(name, false) + + const_get(name, false) end # The current version to be used in new migrations def self.current_version - 1.0 + 2.0 end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 7602e09981a..d016dea224b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -692,6 +692,8 @@ module Gitlab # batch_column_name - option for tables without a primary key, in this case # another unique integer column can be used. Example: :user_id def undo_cleanup_concurrent_column_type_change(table, column, old_type, type_cast_function: nil, batch_column_name: :id, limit: nil) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + temp_column = "#{column}_for_type_change" # Using a descriptive name that includes orinal column's name risks @@ -956,7 +958,7 @@ module Gitlab Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" elsif !migration.finished? raise "Expected batched background migration for the given configuration to be marked as 'finished', " \ - "but it is '#{migration.status}':" \ + "but it is '#{migration.status_name}':" \ "\t#{configuration}" \ "\n\n" \ "Finalize it manualy by running" \ @@ -1639,7 +1641,9 @@ into similar problems in the future (e.g. when new tables are created). old_value = Arel::Nodes::NamedFunction.new(type_cast_function, [old_value]) end - update_column_in_batches(table, new, old_value, batch_column_name: batch_column_name) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + update_column_in_batches(table, new, old_value, batch_column_name: batch_column_name) + end add_not_null_constraint(table, new) unless old_col.null diff --git a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb index b4e31565c60..5a25128f3a9 100644 --- a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb +++ b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb @@ -6,8 +6,6 @@ module Gitlab module RestrictGitlabSchema extend ActiveSupport::Concern - MigrationSkippedError = Class.new(StandardError) - included do class_attribute :allowed_gitlab_schemas end @@ -25,11 +23,8 @@ module Gitlab def migrate(direction) if unmatched_schemas.any? - # TODO: Today skipping migration would raise an exception. - # Ideally, skipped migration should be ignored (not loaded), or softly ignored. - # Read more in: https://gitlab.com/gitlab-org/gitlab/-/issues/355014 - raise MigrationSkippedError, "Current migration is skipped since it modifies "\ - "'#{self.class.allowed_gitlab_schemas}' which is outside of '#{allowed_schemas_for_connection}'" + migration_skipped + return end Gitlab::Database::QueryAnalyzer.instance.within([validator_class]) do @@ -41,6 +36,11 @@ module Gitlab private + def migration_skipped + say "Current migration is skipped since it modifies "\ + "'#{self.class.allowed_gitlab_schemas}' which is outside of '#{allowed_schemas_for_connection}'" + end + def validator_class Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas end diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb index 0e7f6075196..dd426962033 100644 --- a/lib/gitlab/database/migration_helpers/v2.rb +++ b/lib/gitlab/database/migration_helpers/v2.rb @@ -134,6 +134,8 @@ module Gitlab # batch_column_name - option is for tables without primary key, in this # case another unique integer column can be used. Example: :user_id def rename_column_concurrently(table, old_column, new_column, type: nil, batch_column_name: :id) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + setup_renamed_column(__callee__, table, old_column, new_column, type, batch_column_name) with_lock_retries do @@ -181,6 +183,8 @@ module Gitlab # case another unique integer column can be used. Example: :user_id # def undo_cleanup_concurrent_column_rename(table, old_column, new_column, type: nil, batch_column_name: :id) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + setup_renamed_column(__callee__, table, new_column, old_column, type, batch_column_name) with_lock_retries do diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index a2a4a37ab87..0261ade0fe7 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -84,7 +84,7 @@ module Gitlab FROM #{connection.quote_table_name(batch_table_name)} SQL - migration_status = batch_max_value.nil? ? :finished : :active + status_event = batch_max_value.nil? ? :finish : :execute batch_max_value ||= batch_min_value migration = Gitlab::Database::BackgroundMigration::BatchedMigration.new( @@ -98,7 +98,7 @@ module Gitlab batch_class_name: batch_class_name, batch_size: batch_size, sub_batch_size: sub_batch_size, - status: migration_status + status_event: status_event ) # Below `BatchedMigration` attributes were introduced after the diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 9d28db6b886..7c21346007a 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -6,11 +6,8 @@ module Gitlab class Instrumentation STATS_FILENAME = 'migration-stats.json' - attr_reader :observations - def initialize(result_dir:, observer_classes: ::Gitlab::Database::Migrations::Observers.all_observers) @observer_classes = observer_classes - @observations = [] @result_dir = result_dir end @@ -38,15 +35,16 @@ module Gitlab on_each_observer(observers) { |observer| observer.after } on_each_observer(observers) { |observer| observer.record } - record_observation(observation) + record_observation(observation, destination_dir: per_migration_result_dir) end private attr_reader :observer_classes - def record_observation(observation) - @observations << observation + def record_observation(observation, destination_dir:) + stats_file_location = File.join(destination_dir, STATS_FILENAME) + File.write(stats_file_location, observation.to_json) end def on_each_observer(observers, &block) diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index 02645a0d452..3b6f52b43a8 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -6,7 +6,7 @@ module Gitlab class Runner BASE_RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze METADATA_FILENAME = 'metadata.json' - SCHEMA_VERSION = 2 # Version of the output format produced by the runner + SCHEMA_VERSION = 3 # Version of the output format produced by the runner class << self def up @@ -17,6 +17,10 @@ module Gitlab Runner.new(direction: :down, migrations: migrations_for_down, result_dir: BASE_RESULT_DIR.join('down')) end + def background_migrations + TestBackgroundRunner.new(result_dir: BASE_RESULT_DIR.join('background_migrations')) + end + def migration_context @migration_context ||= ApplicationRecord.connection.migration_context end @@ -76,13 +80,8 @@ module Gitlab end end ensure - if instrumentation - stats_filename = File.join(result_dir, Gitlab::Database::Migrations::Instrumentation::STATS_FILENAME) - File.write(stats_filename, instrumentation.observations.to_json) - - metadata_filename = File.join(result_dir, METADATA_FILENAME) - File.write(metadata_filename, { version: SCHEMA_VERSION }.to_json) - end + metadata_filename = File.join(result_dir, METADATA_FILENAME) + File.write(metadata_filename, { version: SCHEMA_VERSION }.to_json) # We clear the cache here to mirror the cache clearing that happens at the end of `db:migrate` tasks # This clearing makes subsequent rake tasks in the same execution pick up database schema changes caused by diff --git a/lib/gitlab/database/migrations/test_background_runner.rb b/lib/gitlab/database/migrations/test_background_runner.rb index 821d68c06c9..74e54d62e05 100644 --- a/lib/gitlab/database/migrations/test_background_runner.rb +++ b/lib/gitlab/database/migrations/test_background_runner.rb @@ -4,12 +4,10 @@ module Gitlab module Database module Migrations class TestBackgroundRunner - # TODO - build a rake task to call this method, and support it in the gitlab-com-database-testing project. - # Until then, we will inject a migration with a very high timestamp during database testing - # that calls this class to run jobs - # See https://gitlab.com/gitlab-org/database-team/gitlab-com-database-testing/-/issues/41 for details + attr_reader :result_dir - def initialize + def initialize(result_dir:) + @result_dir = result_dir @job_coordinator = Gitlab::BackgroundMigration.coordinator_for_database(Gitlab::Database::MAIN_DATABASE_NAME) end @@ -24,18 +22,30 @@ module Gitlab # without .to_f, we do integer division # For example, 3.minutes / 2 == 1.minute whereas 3.minutes / 2.to_f == (1.minute + 30.seconds) duration_per_migration_type = for_duration / jobs_to_run.count.to_f - jobs_to_run.each do |_migration_name, jobs| + jobs_to_run.each do |migration_name, jobs| run_until = duration_per_migration_type.from_now - jobs.shuffle.each do |j| - break if run_until <= Time.current - run_job(j) - end + run_jobs_for_migration(migration_name: migration_name, jobs: jobs, run_until: run_until) end end private + def run_jobs_for_migration(migration_name:, jobs:, run_until:) + per_background_migration_result_dir = File.join(@result_dir, migration_name) + + instrumentation = Instrumentation.new(result_dir: per_background_migration_result_dir) + batch_names = (1..).each.lazy.map { |i| "batch_#{i}"} + + jobs.shuffle.each do |j| + break if run_until <= Time.current + + instrumentation.observe(version: nil, name: batch_names.next, connection: ActiveRecord::Migration.connection) do + run_job(j) + end + end + end + def run_job(job) Gitlab::BackgroundMigration.perform(job.args[0], job.args[1]) end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index e56ffddac4f..034e18ec9f4 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -40,16 +40,20 @@ module Gitlab # 1. The minimum value for the partitioning column in the table # 2. If no data is present yet, the current month def partition_table_by_date(table_name, column_name, min_date: nil, max_date: nil) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + assert_table_is_allowed(table_name) assert_not_in_transaction_block(scope: ERROR_SCOPE) max_date ||= Date.today + 1.month - min_date ||= connection.select_one(<<~SQL)['minimum'] || max_date - 1.month - SELECT date_trunc('MONTH', MIN(#{column_name})) AS minimum - FROM #{table_name} - SQL + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + min_date ||= connection.select_one(<<~SQL)['minimum'] || max_date - 1.month + SELECT date_trunc('MONTH', MIN(#{column_name})) AS minimum + FROM #{table_name} + SQL + end raise "max_date #{max_date} must be greater than min_date #{min_date}" if min_date >= max_date @@ -154,6 +158,8 @@ module Gitlab # finalize_backfilling_partitioned_table :audit_events # def finalize_backfilling_partitioned_table(table_name) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! + assert_table_is_allowed(table_name) assert_not_in_transaction_block(scope: ERROR_SCOPE) @@ -170,8 +176,10 @@ module Gitlab primary_key = connection.primary_key(table_name) copy_missed_records(table_name, partitioned_table_name, primary_key) - disable_statement_timeout do - execute("VACUUM FREEZE ANALYZE #{partitioned_table_name}") + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + disable_statement_timeout do + execute("VACUUM FREEZE ANALYZE #{partitioned_table_name}") + end end end diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb index 06e2b114c91..391375d472f 100644 --- a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb +++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb @@ -27,9 +27,15 @@ module Gitlab # to reduce amount of labels sort schemas used gitlab_schemas = gitlab_schemas.to_a.sort.join(",") + # Temporary feature to observe relation of `gitlab_schemas` to `db_config_name` + # depending on primary model + ci_dedicated_primary_connection = ::Ci::ApplicationRecord.connection_class? && + ::Ci::ApplicationRecord.load_balancer.configuration.use_dedicated_connection? + schemas_metrics.increment({ gitlab_schemas: gitlab_schemas, - db_config_name: db_config_name + db_config_name: db_config_name, + ci_dedicated_primary_connection: ci_dedicated_primary_connection }) end diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb index ab40ba5d59b..3f0176cb654 100644 --- a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb +++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb @@ -69,8 +69,10 @@ module Gitlab schemas = self.dml_schemas(tables) if (schemas - self.allowed_gitlab_schemas).any? - raise DMLAccessDeniedError, "Select/DML queries (SELECT/UPDATE/DELETE) do access '#{tables}' (#{schemas.to_a}) " \ - "which is outside of list of allowed schemas: '#{self.allowed_gitlab_schemas}'." + raise DMLAccessDeniedError, \ + "Select/DML queries (SELECT/UPDATE/DELETE) do access '#{tables}' (#{schemas.to_a}) " \ + "which is outside of list of allowed schemas: '#{self.allowed_gitlab_schemas}'. " \ + "#{documentation_url}" end end @@ -93,11 +95,19 @@ module Gitlab end def raise_dml_not_allowed_error(message) - raise DMLNotAllowedError, "Select/DML queries (SELECT/UPDATE/DELETE) are disallowed in the DDL (structure) mode. #{message}" + raise DMLNotAllowedError, \ + "Select/DML queries (SELECT/UPDATE/DELETE) are disallowed in the DDL (structure) mode. " \ + "#{message}. #{documentation_url}" \ end def raise_ddl_not_allowed_error(message) - raise DDLNotAllowedError, "DDL queries (structure) are disallowed in the Select/DML (SELECT/UPDATE/DELETE) mode. #{message}" + raise DDLNotAllowedError, \ + "DDL queries (structure) are disallowed in the Select/DML (SELECT/UPDATE/DELETE) mode. " \ + "#{message}. #{documentation_url}" + end + + def documentation_url + "For more information visit: https://docs.gitlab.com/ee/development/database/migrations_for_multiple_databases.html" end end end diff --git a/lib/gitlab/database/reindexing/grafana_notifier.rb b/lib/gitlab/database/reindexing/grafana_notifier.rb index f4ea59deb50..ece9327b658 100644 --- a/lib/gitlab/database/reindexing/grafana_notifier.rb +++ b/lib/gitlab/database/reindexing/grafana_notifier.rb @@ -5,10 +5,10 @@ module Gitlab module Reindexing # This can be used to send annotations for reindexing to a Grafana API class GrafanaNotifier - def initialize(api_key = ENV['GITLAB_GRAFANA_API_KEY'], api_url = ENV['GITLAB_GRAFANA_API_URL'], additional_tag = ENV['GITLAB_REINDEXING_GRAFANA_TAG'] || Rails.env) - @api_key = api_key - @api_url = api_url - @additional_tag = additional_tag + def initialize(api_key: nil, api_url: nil, additional_tag: nil) + @api_key = api_key || default_api_key + @api_url = api_url || default_api_url + @additional_tag = additional_tag || default_additional_tag end def notify_start(action) @@ -35,10 +35,22 @@ module Gitlab private + def default_api_key + Gitlab::CurrentSettings.database_grafana_api_key || ENV['GITLAB_GRAFANA_API_KEY'] + end + + def default_api_url + Gitlab::CurrentSettings.database_grafana_api_url || ENV['GITLAB_GRAFANA_API_URL'] + end + + def default_additional_tag + Gitlab::CurrentSettings.database_grafana_tag || ENV['GITLAB_REINDEXING_GRAFANA_TAG'] || Rails.env + end + def base_payload(action) { time: (action.action_start.utc.to_f * 1000).to_i, - tags: ['reindex', @additional_tag, action.index.tablename, action.index.name].compact + tags: ['reindex', @additional_tag.presence, action.index.tablename, action.index.name].compact } end diff --git a/lib/gitlab/diff/custom_diff.rb b/lib/gitlab/diff/custom_diff.rb index af1fd8fb03e..860f87a28a3 100644 --- a/lib/gitlab/diff/custom_diff.rb +++ b/lib/gitlab/diff/custom_diff.rb @@ -2,17 +2,29 @@ module Gitlab module Diff module CustomDiff + RENDERED_TIMEOUT_BACKGROUND = 20.seconds + RENDERED_TIMEOUT_FOREGROUND = 1.5.seconds + BACKGROUND_EXECUTION = 'background' + FOREGROUND_EXECUTION = 'foreground' + LOG_IPYNBDIFF_GENERATED = 'IPYNB_DIFF_GENERATED' + LOG_IPYNBDIFF_TIMEOUT = 'IPYNB_DIFF_TIMEOUT' + LOG_IPYNBDIFF_INVALID = 'IPYNB_DIFF_INVALID' + class << self def preprocess_before_diff(path, old_blob, new_blob) return unless path.ends_with? '.ipynb' - transformed_diff(old_blob&.data, new_blob&.data)&.tap do - transformed_for_diff(new_blob, old_blob) - Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' }) + Timeout.timeout(timeout_time) do + transformed_diff(old_blob&.data, new_blob&.data)&.tap do + transformed_for_diff(new_blob, old_blob) + log_event(LOG_IPYNBDIFF_GENERATED) + end end + rescue Timeout::Error => e + rendered_timeout.increment(source: execution_source) + log_event(LOG_IPYNBDIFF_TIMEOUT, e) rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e - Gitlab::ErrorTracking.log_exception(e) - nil + log_event(LOG_IPYNBDIFF_INVALID, e) end def transformed_diff(before, after) @@ -50,6 +62,27 @@ module Gitlab blobs_with_transformed_diffs[b] = true if b end end + + def rendered_timeout + @rendered_timeout ||= Gitlab::Metrics.counter( + :ipynb_semantic_diff_timeouts_total, + 'Counts the times notebook rendering timed out' + ) + end + + def timeout_time + Gitlab::Runtime.sidekiq? ? RENDERED_TIMEOUT_BACKGROUND : RENDERED_TIMEOUT_FOREGROUND + end + + def execution_source + Gitlab::Runtime.sidekiq? ? BACKGROUND_EXECUTION : FOREGROUND_EXECUTION + end + + def log_event(message, error = nil) + Gitlab::AppLogger.info({ message: message }) + Gitlab::ErrorTracking.track_exception(error) if error + nil + end end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 89822af2455..61bb0c797b4 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -44,7 +44,13 @@ module Gitlab new_blob_lazy old_blob_lazy - diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff unless use_renderable_diff? + if use_semantic_ipynb_diff? && !use_renderable_diff? + diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff + end + end + + def use_semantic_ipynb_diff? + strong_memoize(:_use_semantic_ipynb_diff) { Feature.enabled?(:ipynb_semantic_diff, repository.project, default_enabled: :yaml) } end def use_renderable_diff? @@ -375,7 +381,7 @@ module Gitlab end def rendered - return unless use_renderable_diff? && ipynb? + return unless use_semantic_ipynb_diff? && use_renderable_diff? && ipynb? && modified_file? && !too_large? strong_memoize(:rendered) { Rendered::Notebook::DiffFile.new(self) } end @@ -410,7 +416,7 @@ module Gitlab end def ipynb? - modified_file? && file_path.ends_with?('.ipynb') + file_path.ends_with?('.ipynb') end # We can't use Object#try because Blob doesn't inherit from Object, but diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index c2b834c71b5..316a0d2815a 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -9,8 +9,8 @@ module Gitlab SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze attr_reader :marker_ranges - attr_writer :text, :rich_text, :discussable - attr_accessor :index, :type, :old_pos, :new_pos, :line_code + attr_writer :text, :rich_text + attr_accessor :index, :old_pos, :new_pos, :line_code, :type def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) @text = text @@ -24,9 +24,7 @@ module Gitlab # When line code is not provided from cache store we build it # using the parent_file(Diff::File or Conflict::File). @line_code = line_code || calculate_line_code - @marker_ranges = [] - @discussable = true end def self.init_from_hash(hash) @@ -81,23 +79,28 @@ module Gitlab end def added? - %w[new new-nonewline].include?(type) + %w[new new-nonewline new-nomappinginraw].include?(type) end def removed? - %w[old old-nonewline].include?(type) + %w[old old-nonewline old-nomappinginraw].include?(type) end def meta? %w[match new-nonewline old-nonewline].include?(type) end + def has_mapping_in_raw? + # Used for rendered diff, when the displayed line doesn't have a matching line in the raw diff + !type&.ends_with?('nomappinginraw') + end + def match? type == :match end def discussable? - @discussable && !meta? + has_mapping_in_raw? && !meta? end def suggestible? diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index 77b65fea726..cbfc20d3d62 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -44,7 +44,7 @@ module Gitlab free_right_index = nil i += 1 end - elsif line.meta? || line.unchanged? + elsif line.meta? || line.unchanged? || !line.has_mapping_in_raw? # line in the right panel is the same as in the left one lines << { left: line, diff --git a/lib/gitlab/diff/rendered/notebook/diff_file.rb b/lib/gitlab/diff/rendered/notebook/diff_file.rb index e700e730f20..cf97569ca31 100644 --- a/lib/gitlab/diff/rendered/notebook/diff_file.rb +++ b/lib/gitlab/diff/rendered/notebook/diff_file.rb @@ -6,6 +6,14 @@ module Gitlab include Gitlab::Utils::StrongMemoize class DiffFile < Gitlab::Diff::File + RENDERED_TIMEOUT_BACKGROUND = 10.seconds + RENDERED_TIMEOUT_FOREGROUND = 1.5.seconds + BACKGROUND_EXECUTION = 'background' + FOREGROUND_EXECUTION = 'foreground' + LOG_IPYNBDIFF_GENERATED = 'IPYNB_DIFF_GENERATED' + LOG_IPYNBDIFF_TIMEOUT = 'IPYNB_DIFF_TIMEOUT' + LOG_IPYNBDIFF_INVALID = 'IPYNB_DIFF_INVALID' + attr_reader :source_diff delegate :repository, :diff_refs, :fallback_diff_refs, :unfolded, :unique_identifier, @@ -52,14 +60,17 @@ module Gitlab def notebook_diff strong_memoize(:notebook_diff) do - Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' }) - - IpynbDiff.diff(source_diff.old_blob&.data, source_diff.new_blob&.data, - raise_if_invalid_nb: true, - diffy_opts: { include_diff_info: true }) + Timeout.timeout(timeout_time) do + IpynbDiff.diff(source_diff.old_blob&.data, source_diff.new_blob&.data, + raise_if_invalid_nb: true, diffy_opts: { include_diff_info: true })&.tap do + log_event(LOG_IPYNBDIFF_GENERATED) + end + end + rescue Timeout::Error => e + rendered_timeout.increment(source: Gitlab::Runtime.sidekiq? ? BACKGROUND_EXECUTION : FOREGROUND_EXECUTION) + log_event(LOG_IPYNBDIFF_TIMEOUT, e) rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e - Gitlab::ErrorTracking.log_exception(e) - nil + log_event(LOG_IPYNBDIFF_INVALID, e) end end @@ -87,10 +98,7 @@ module Gitlab line.new_pos = removal_line_maps[line.old_pos] if line.new_pos == 0 && line.old_pos != 0 # Lines that do not appear on the original diff should not be commentable - - unless addition_line_maps[line.new_pos] || removal_line_maps[line.old_pos] - line.discussable = false - end + line.type = "#{line.type || 'unchanged'}-nomappinginraw" unless addition_line_maps[line.new_pos] || removal_line_maps[line.old_pos] line.line_code = line_code(line) line @@ -113,12 +121,29 @@ module Gitlab additions = {} source_diff.highlighted_diff_lines.each do |line| - removals[line.old_pos] = line.new_pos - additions[line.new_pos] = line.old_pos + removals[line.old_pos] = line.new_pos unless source_diff.new_file? + additions[line.new_pos] = line.old_pos unless source_diff.deleted_file? end [removals, additions] end + + def rendered_timeout + @rendered_timeout ||= Gitlab::Metrics.counter( + :ipynb_semantic_diff_timeouts_total, + 'Counts the times notebook diff rendering timed out' + ) + end + + def timeout_time + Gitlab::Runtime.sidekiq? ? RENDERED_TIMEOUT_BACKGROUND : RENDERED_TIMEOUT_FOREGROUND + end + + def log_event(message, error = nil) + Gitlab::AppLogger.info({ message: message }) + Gitlab::ErrorTracking.track_exception(error) if error + nil + end end end end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index bb57494c729..71b1d4ed8f9 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -34,7 +34,7 @@ module Gitlab create_issue_or_note - if issue_creator_address + if from_address add_email_participant send_thank_you_email unless reply_email? end @@ -98,7 +98,7 @@ module Gitlab title: mail.subject, description: message_including_template, confidential: true, - external_author: external_author + external_author: from_address }, spam_params: nil ).execute @@ -176,22 +176,8 @@ module Gitlab ).execute end - def issue_creator_address - reply_to_address || from_address - end - def from_address - mail.from.first || mail.sender - end - - def reply_to_address - (mail.reply_to || []).first - end - - def external_author - return issue_creator_address unless reply_to_address && from_address - - _("%{from_address} (reply to: %{reply_to_address})") % { from_address: from_address, reply_to_address: reply_to_address } + (mail.reply_to || []).first || mail.from.first || mail.sender end def can_handle_legacy_format? @@ -205,7 +191,7 @@ module Gitlab def add_email_participant return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project) - @issue.issue_email_participants.create(email: issue_creator_address) + @issue.issue_email_participants.create(email: from_address) end end end diff --git a/lib/gitlab/email/message/in_product_marketing.rb b/lib/gitlab/email/message/in_product_marketing.rb index ac9585bcd1a..bd2c91755c8 100644 --- a/lib/gitlab/email/message/in_product_marketing.rb +++ b/lib/gitlab/email/message/in_product_marketing.rb @@ -7,7 +7,7 @@ module Gitlab UnknownTrackError = Class.new(StandardError) def self.for(track) - valid_tracks = [Namespaces::InviteTeamEmailService::TRACK, Namespaces::InProductMarketingEmailsService::TRACKS.keys].flatten + valid_tracks = Namespaces::InProductMarketingEmailsService::TRACKS.keys raise UnknownTrackError unless valid_tracks.include?(track) "Gitlab::Email::Message::InProductMarketing::#{track.to_s.classify}".constantize diff --git a/lib/gitlab/email/message/in_product_marketing/invite_team.rb b/lib/gitlab/email/message/in_product_marketing/invite_team.rb deleted file mode 100644 index e9334b687f4..00000000000 --- a/lib/gitlab/email/message/in_product_marketing/invite_team.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - class InviteTeam < Base - def subject_line - s_('InProductMarketing|Invite your teammates to GitLab') - end - - def tagline - '' - end - - def title - s_('InProductMarketing|GitLab is better with teammates to help out!') - end - - def subtitle - '' - end - - def body_line1 - s_('InProductMarketing|Invite your teammates today and build better code together. You can even assign tasks to new teammates such as setting up CI/CD, to help get projects up and running.') - end - - def body_line2 - '' - end - - def cta_text - s_('InProductMarketing|Invite your teammates to help') - end - - def logo_path - 'mailers/in_product_marketing/team-0.png' - end - - def series? - false - end - - private - - def validate_series! - raise ArgumentError, "Only one email is sent for this track. Value of `series` should be 0." unless @series == 0 - end - end - end - end - end -end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index 3c5d223b106..f539d627dcb 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -46,12 +46,13 @@ module Gitlab def custom_emoji_tag(name, image_source) data = { - name: name + name: name, + fallback_src: image_source, + unicode_version: 'custom' # Prevents frontend to check for Unicode support } + options = { title: name, data: data } - ActionController::Base.helpers.content_tag('gl-emoji', title: name, data: data) do - emoji_image_tag(name, image_source).html_safe - end + ActionController::Base.helpers.content_tag('gl-emoji', "", options) end end end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 2e0060c7c18..f26ab6e3ed1 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -15,6 +15,8 @@ module Gitlab # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 ENCODING_CONFIDENCE_THRESHOLD = 50 + UNICODE_REPLACEMENT_CHARACTER = "�" + def encode!(message) message = force_encode_utf8(message) return message if message.valid_encoding? @@ -65,6 +67,10 @@ module Gitlab message.encode(Encoding::UTF_8, invalid: :replace, undef: :replace) end + def encode_utf8_with_replacement_character(data) + encode_utf8(data, replace: UNICODE_REPLACEMENT_CHARACTER) + end + def encode_utf8(message, replace: "") message = force_encode_utf8(message) return message if message.valid_encoding? @@ -99,6 +105,35 @@ module Gitlab io.tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } end + ESCAPED_CHARS = { + "a" => "\a", "b" => "\b", "e" => "\e", "f" => "\f", + "n" => "\n", "r" => "\r", "t" => "\t", "v" => "\v", + "\"" => "\"" + }.freeze + + # rubocop:disable Style/AsciiComments + # `unquote_path` decode filepaths that are returned by some git commands. + # The path may be returned in double-quotes if it contains special characters, + # that are encoded in octal. Also, some characters (see `ESCAPED_CHARS`) are escaped. + # eg. "\311\240\304\253\305\247\305\200\310\247\306\200" (quotes included) is decoded as É Ä«Å§Å€È§Æ€ + # + # Based on `unquote_c_style` from git source + # https://github.com/git/git/blob/v2.35.1/quote.c#L399 + # rubocop:enable Style/AsciiComments + def unquote_path(filename) + return filename unless filename[0] == '"' + + filename = filename[1..-2].gsub(/\\(?:([#{ESCAPED_CHARS.keys.join}\\])|(\d{3}))/) do + if c = Regexp.last_match(1) + c == "\\" ? "\\" : ESCAPED_CHARS[c] + elsif c = Regexp.last_match(2) + c.to_i(8).chr + end + end + + filename.force_encoding("UTF-8") + end + private def force_encode_utf8(message) diff --git a/lib/gitlab/experiment/rollout/feature.rb b/lib/gitlab/experiment/rollout/feature.rb index 70c363877b1..4bef92f5c23 100644 --- a/lib/gitlab/experiment/rollout/feature.rb +++ b/lib/gitlab/experiment/rollout/feature.rb @@ -28,13 +28,10 @@ module Gitlab # If the `Feature.enabled?` check is false, we return nil implicitly, # which will assign the control. Otherwise we call super, which will # assign a variant evenly, or based on our provided distribution rules. - def execute_assigment + def execute_assignment super if ::Feature.enabled?(feature_flag_name, self, type: :experiment, default_enabled: :yaml) end - # NOTE: There's a typo in the name of this method that we'll fix up. - alias_method :execute_assignment, :execute_assigment - # This is what's provided to the `Feature.enabled?` call that will be # used to determine experiment inclusion. An experiment may provide an # override for this method to make the experiment work on user, group, diff --git a/lib/gitlab/fips.rb b/lib/gitlab/fips.rb index 1dd363ceb17..97813f13a91 100644 --- a/lib/gitlab/fips.rb +++ b/lib/gitlab/fips.rb @@ -5,6 +5,17 @@ module Gitlab class FIPS # A simple utility class for FIPS-related helpers + Technology = Gitlab::SSHPublicKey::Technology + + SSH_KEY_TECHNOLOGIES = [ + Technology.new(:rsa, SSHData::PublicKey::RSA, [3072, 4096], %w(ssh-rsa)), + Technology.new(:dsa, SSHData::PublicKey::DSA, [], %w(ssh-dss)), + Technology.new(:ecdsa, SSHData::PublicKey::ECDSA, [256, 384, 521], %w(ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521)), + Technology.new(:ed25519, SSHData::PublicKey::ED25519, [256], %w(ssh-ed25519)), + Technology.new(:ecdsa_sk, SSHData::PublicKey::SKECDSA, [256], %w(sk-ecdsa-sha2-nistp256@openssh.com)), + Technology.new(:ed25519_sk, SSHData::PublicKey::SKED25519, [256], %w(sk-ssh-ed25519@openssh.com)) + ].freeze + class << self # Returns whether we should be running in FIPS mode or not # diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 08321d5fda6..82ef7eed56a 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -37,7 +37,7 @@ module Gitlab if was_embedded?(markdown) moved_markdown else - moved_markdown.sub(/\A!/, "") + moved_markdown.delete_prefix('!') end end end diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 5669a65cbd9..30977adaea1 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -5,35 +5,45 @@ module Gitlab class Blame include Gitlab::EncodingHelper - attr_reader :lines, :blames + attr_reader :lines, :blames, :range - def initialize(repository, sha, path) + def initialize(repository, sha, path, range: nil) @repo = repository @sha = sha @path = path + @range = range @lines = [] @blames = load_blame end def each @blames.each do |blame| - yield(blame.commit, blame.line) + yield(blame.commit, blame.line, blame.previous_path) end end private + def range_spec + "#{range.first},#{range.last}" if range + end + def load_blame - output = encode_utf8(@repo.gitaly_commit_client.raw_blame(@sha, @path)) + output = encode_utf8( + @repo.gitaly_commit_client.raw_blame(@sha, @path, range: range_spec) + ) process_raw_blame(output) end def process_raw_blame(output) + start_line = nil lines = [] final = [] info = {} commits = {} + commit_id = nil + previous_paths = {} # process the output output.split("\n").each do |line| @@ -45,6 +55,15 @@ module Gitlab commit_id = m[1] commits[commit_id] = nil unless commits.key?(commit_id) info[m[3].to_i] = [commit_id, m[2].to_i] + + # Assumption: the first line returned by git blame is lowest-numbered + # This is true unless we start passing it `--incremental`. + start_line = m[3].to_i if start_line.nil? + elsif line.start_with?("previous ") + # previous 1485b69e7b839a21436e81be6d3aa70def5ed341 initial-commit + # previous 9521e52704ee6100e7d2a76896a4ef0eb53ff1b8 "\303\2511\\\303\251\\303\\251\n" + # ^ char index 50 + previous_paths[commit_id] = unquote_path(line[50..]) end end @@ -54,7 +73,13 @@ module Gitlab # get it together info.sort.each do |lineno, (commit_id, old_lineno)| - final << BlameLine.new(lineno, old_lineno, commits[commit_id], lines[lineno - 1]) + final << BlameLine.new( + lineno, + old_lineno, + commits[commit_id], + lines[lineno - start_line], + previous_paths[commit_id] + ) end @lines = final @@ -62,13 +87,14 @@ module Gitlab end class BlameLine - attr_accessor :lineno, :oldlineno, :commit, :line + attr_accessor :lineno, :oldlineno, :commit, :line, :previous_path - def initialize(lineno, oldlineno, commit, line) + def initialize(lineno, oldlineno, commit, line, previous_path) @lineno = lineno @oldlineno = oldlineno @commit = commit @line = line + @previous_path = previous_path end end end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 8325eadce2f..a66517b4ca0 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -140,7 +140,7 @@ module Gitlab text.start_with?(BINARY_NOTICE_PATTERN) end end - def initialize(raw_diff, expanded: true) + def initialize(raw_diff, expanded: true, replace_invalid_utf8_chars: true) @expanded = expanded case raw_diff @@ -157,6 +157,8 @@ module Gitlab else raise "Invalid raw diff type: #{raw_diff.class}" end + + encode_diff_to_utf8(replace_invalid_utf8_chars) end def to_hash @@ -227,6 +229,13 @@ module Gitlab private + def encode_diff_to_utf8(replace_invalid_utf8_chars) + return unless Feature.enabled?(:convert_diff_to_utf8_with_replacement_symbol, default_enabled: :yaml) + return unless replace_invalid_utf8_chars && !detect_binary?(@diff) + + @diff = Gitlab::EncodingHelper.encode_utf8_with_replacement_character(@diff) + end + def init_from_hash(hash) raw_diff = hash.symbolize_keys diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 24b67424f28..0ffe8bee953 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -9,8 +9,6 @@ module Gitlab attr_reader :limits - delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits - def self.default_limits { max_files: ::Commit.diff_safe_max_files, max_lines: ::Commit.diff_safe_max_lines } end @@ -26,8 +24,7 @@ module Gitlab limits[:safe_max_lines] = [limits[:max_lines], defaults[:max_lines]].min limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes - - OpenStruct.new(limits) + limits end def initialize(iterator, options = {}) @@ -140,11 +137,11 @@ module Gitlab end def over_safe_limits?(files) - if files >= safe_max_files + if files >= limits[:safe_max_files] @collapsed_safe_files = true - elsif @line_count > safe_max_lines + elsif @line_count > limits[:safe_max_lines] @collapsed_safe_lines = true - elsif @byte_count >= safe_max_bytes + elsif @byte_count >= limits[:safe_max_bytes] @collapsed_safe_bytes = true end @@ -179,7 +176,7 @@ module Gitlab @iterator.each_with_index do |raw, iterator_index| @empty = false - if @enforce_limits && i >= max_files + if @enforce_limits && i >= limits[:max_files] @overflow = true @overflow_max_files = true break @@ -194,7 +191,7 @@ module Gitlab @line_count += diff.line_count @byte_count += diff.diff.bytesize - if @enforce_limits && @line_count >= max_lines + if @enforce_limits && @line_count >= limits[:max_lines] # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true @@ -202,7 +199,7 @@ module Gitlab break end - if @enforce_limits && @byte_count >= max_bytes + if @enforce_limits && @byte_count >= limits[:max_bytes] # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 47cfb483509..1d7966a11ed 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -24,7 +24,7 @@ module Gitlab # Ex. # Ref.extract_branch_name('refs/heads/master') #=> 'master' def self.extract_branch_name(str) - str.gsub(%r{\Arefs/heads/}, '') + str.delete_prefix('refs/heads/') end def initialize(repository, name, target, dereferenced_target) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 1492ea1ce76..ab365069adf 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -841,11 +841,11 @@ module Gitlab end end - def import_repository(url) + def import_repository(url, http_authorization_header: '', mirror: false) raise ArgumentError, "don't use disk paths with import_repository: #{url.inspect}" if url.start_with?('.', '/') wrapped_gitaly_errors do - gitaly_repository_client.import_repository(url) + gitaly_repository_client.import_repository(url, http_authorization_header: http_authorization_header, mirror: mirror) end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 0e3f9c2598d..4fe5c8df36f 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -315,11 +315,12 @@ module Gitlab response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } } end - def raw_blame(revision, path) + def raw_blame(revision, path, range:) request = Gitaly::RawBlameRequest.new( repository: @gitaly_repo, revision: encode_binary(revision), - path: encode_binary(path) + path: encode_binary(path), + range: (encode_binary(range) if range) ) response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) @@ -466,7 +467,7 @@ module Gitlab request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) request_params[:enforce_limits] = options.fetch(:limits, true) request_params[:collapse_diffs] = !options.fetch(:expanded, true) - request_params.merge!(Gitlab::Git::DiffCollection.limits(options).to_h) + request_params.merge!(Gitlab::Git::DiffCollection.limits(options)) request = Gitaly::CommitDiffRequest.new(request_params) response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 5c447dfd417..1e199a55b5a 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -145,10 +145,12 @@ module Gitlab ) end - def import_repository(source) + def import_repository(source, http_authorization_header: '', mirror: false) request = Gitaly::CreateRepositoryFromURLRequest.new( repository: @gitaly_repo, - url: source + url: source, + http_authorization_header: http_authorization_header, + mirror: mirror ) GitalyClient.call( diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb index 7ce88280209..8873db24118 100644 --- a/lib/gitlab/github_import/object_counter.rb +++ b/lib/gitlab/github_import/object_counter.rb @@ -24,6 +24,8 @@ module Gitlab increment_project_counter(project, object_type, operation, integer) increment_global_counter(object_type, operation, integer) + + project.import_state&.expire_etag_cache end def summary(project) diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 4dec9543a13..97de2a49e72 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -72,7 +72,7 @@ module Gitlab # Imports all objects in parallel by scheduling a Sidekiq job for every # individual object. def parallel_import - if Feature.enabled?(:spread_parallel_import, default_enabled: :yaml) && parallel_import_batch.present? + if parallel_import_batch.present? spread_parallel_import else parallel_import_deprecated @@ -209,7 +209,11 @@ module Gitlab # Default batch settings for parallel import (can be redefined in Importer classes) # Example: { size: 100, delay: 1.minute } def parallel_import_batch - {} + if Feature.enabled?(:distribute_github_parallel_import, default_enabled: :yaml) + { size: 1000, delay: 1.minute } + else + {} + end end def abort_on_failure diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 9f18513f066..3c85d56874f 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -53,13 +53,13 @@ module Gitlab # made globally available to the frontend push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) - push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml) push_frontend_feature_flag(:new_header_search, default_enabled: :yaml) push_frontend_feature_flag(:bootstrap_confirmation_modals, default_enabled: :yaml) push_frontend_feature_flag(:sandboxed_mermaid, default_enabled: :yaml) push_frontend_feature_flag(:source_editor_toolbar, default_enabled: :yaml) push_frontend_feature_flag(:gl_avatar_for_all_user_avatars, default_enabled: :yaml) push_frontend_feature_flag(:mr_attention_requests, default_enabled: :yaml) + push_frontend_feature_flag(:markdown_continue_lists, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. @@ -73,6 +73,15 @@ module Gitlab push_to_gon_attributes(:features, name, enabled) end + # Exposes the state of a feature flag to the frontend code. + # Can be used for more complex feature flag checks. + # + # name - The name of the feature flag, e.g. `my_feature`. + # enabled - Boolean to be pushed directly to the frontend. Should be fetched by checking a feature flag. + def push_force_frontend_feature_flag(name, enabled) + push_to_gon_attributes(:features, name, !!enabled) + end + def push_to_gon_attributes(key, name, enabled) var_name = name.to_s.camelize(:lower) # Here the `true` argument signals gon that the value should be merged diff --git a/lib/gitlab/graphql/deprecation.rb b/lib/gitlab/graphql/deprecation.rb index 20068758502..3335e511714 100644 --- a/lib/gitlab/graphql/deprecation.rb +++ b/lib/gitlab/graphql/deprecation.rb @@ -5,7 +5,7 @@ module Gitlab class Deprecation REASONS = { renamed: 'This was renamed.', - discouraged: 'Use of this is not recommended.' + alpha: 'This feature is in Alpha, and can be removed or changed at any point.' }.freeze include ActiveModel::Validations diff --git a/lib/gitlab/graphql/known_operations.rb b/lib/gitlab/graphql/known_operations.rb index ead52935945..a551c9bb6da 100644 --- a/lib/gitlab/graphql/known_operations.rb +++ b/lib/gitlab/graphql/known_operations.rb @@ -14,7 +14,6 @@ module Gitlab end end - ANONYMOUS = Operation.new("anonymous").freeze UNKNOWN = Operation.new("unknown").freeze def self.default @@ -24,7 +23,7 @@ module Gitlab def initialize(operation_names) @operation_hash = operation_names .map { |name| Operation.new(name).freeze } - .concat([ANONYMOUS, UNKNOWN]) + .concat([UNKNOWN]) .index_by(&:name) end @@ -32,7 +31,7 @@ module Gitlab def from_query(query) operation_name = query.selected_operation_name - return ANONYMOUS unless operation_name + return UNKNOWN unless operation_name @operation_hash[operation_name] || UNKNOWN end diff --git a/lib/gitlab/graphql/pagination/active_record_array_connection.rb b/lib/gitlab/graphql/pagination/active_record_array_connection.rb new file mode 100644 index 00000000000..9e40f79b2fd --- /dev/null +++ b/lib/gitlab/graphql/pagination/active_record_array_connection.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +# Connection for an array of Active Record instances. +# Resolvers needs to handle cursors (before and after). +# This connection will handle (first and last). +# Supports batch loaded items. +# Expects the array to use a fixed DESC order. This is similar to +# ExternallyPaginatedArrayConnection. +module Gitlab + module Graphql + module Pagination + class ActiveRecordArrayConnection < GraphQL::Pagination::ArrayConnection + include ::Gitlab::Graphql::ConnectionCollectionMethods + prepend ::Gitlab::Graphql::ConnectionRedaction + + delegate :<<, to: :items + + def nodes + load_nodes + + @nodes + end + + def next_page? + load_nodes + + if before + true + elsif first + limit_value < items.size + else + false + end + end + + def previous_page? + load_nodes + + if after + true + elsif last + limit_value < items.size + else + false + end + end + + # see https://graphql-ruby.org/pagination/custom_connections#connection-wrapper + alias_method :has_next_page, :next_page? + alias_method :has_previous_page, :previous_page? + + def cursor_for(item) + # item could be a batch loaded item. Sync it to have the id. + cursor = { 'id' => Gitlab::Graphql::Lazy.force(item).id.to_s } + encode(cursor.to_json) + end + + # Part of the implied interface for default objects for BatchLoader: objects must be clonable + def dup + self.class.new( + items.dup, + first: first, + after: after, + max_page_size: max_page_size, + last: last, + before: before + ) + end + + private + + def limit_value + # note: only first _or_ last can be specified, not both + @limit_value ||= [first, last, max_page_size].compact.min + end + + def load_nodes + @nodes ||= begin + limited_nodes = items + + limited_nodes = limited_nodes.first(first) if first + limited_nodes = limited_nodes.last(last) if last + + limited_nodes + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index 61903c566f0..c284160e539 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -14,10 +14,6 @@ # Issue.order(created_at: :asc).order(:id) # Issue.order(due_date: :asc) # -# You can also use `Gitlab::Database.nulls_last_order`: -# -# Issue.reorder(::Gitlab::Database.nulls_last_order('due_date', 'DESC')) -# # It will tolerate non-attribute ordering, but only attributes determine the cursor. # For example, this is legitimate: # diff --git a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb index e8335a3c79c..bf9b73d918a 100644 --- a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb +++ b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb @@ -73,9 +73,24 @@ module Gitlab strong_memoize(:generic_keyset_pagination_items) do rebuilt_items_with_keyset_order, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(original_items) - success ? rebuilt_items_with_keyset_order : original_items + if success + rebuilt_items_with_keyset_order + else + if original_items.is_a?(ActiveRecord::Relation) + old_keyset_pagination_usage.increment({ model: original_items.model.to_s }) + end + + original_items + end end end + + def old_keyset_pagination_usage + @old_keyset_pagination_usage ||= Gitlab::Metrics.counter( + :old_keyset_pagination_usage, + 'The number of times the old keyset pagination code was used' + ) + end end end end diff --git a/lib/gitlab/graphql/project/dast_profile_connection_extension.rb b/lib/gitlab/graphql/project/dast_profile_connection_extension.rb index a3c3f2f2b7e..45f90de2f17 100644 --- a/lib/gitlab/graphql/project/dast_profile_connection_extension.rb +++ b/lib/gitlab/graphql/project/dast_profile_connection_extension.rb @@ -2,7 +2,7 @@ module Gitlab module Graphql module Project - class DastProfileConnectionExtension < GraphQL::Schema::Field::ConnectionExtension + class DastProfileConnectionExtension < GraphQL::Schema::FieldExtension def after_resolve(value:, object:, context:, **rest) preload_authorizations(context[:project_dast_profiles]) context[:project_dast_profiles] = nil diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb index 5c8aa5050ed..add9e880475 100644 --- a/lib/gitlab/hook_data/issuable_builder.rb +++ b/lib/gitlab/hook_data/issuable_builder.rb @@ -13,7 +13,7 @@ module Gitlab event_type: event_type, user: user.hook_attrs, project: issuable.project.hook_attrs, - object_attributes: issuable.hook_attrs, + object_attributes: issuable_builder.new(issuable).build, labels: issuable.labels.map(&:hook_attrs), changes: final_changes(changes.slice(*safe_keys)), # DEPRECATED @@ -53,10 +53,7 @@ module Gitlab end def final_changes(changes_hash) - changes_hash.reduce({}) do |hash, (key, changes_array)| - hash[key] = Hash[CHANGES_KEYS.zip(changes_array)] - hash - end + changes_hash.transform_values { |changes_array| Hash[CHANGES_KEYS.zip(changes_array)] } end end end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index aaca16d8d7c..06ddd65d075 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -60,6 +60,7 @@ module Gitlab human_time_estimate: merge_request.human_time_estimate, assignee_ids: merge_request.assignee_ids, assignee_id: merge_request.assignee_ids.first, # This key is deprecated + labels: merge_request.labels_hook_attrs, state: merge_request.state, # This key is deprecated blocking_discussions_resolved: merge_request.mergeable_discussions_state? } diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 002708beb3c..7b1657d3854 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -29,17 +29,13 @@ module Gitlab http = super http.hostname_override = hostname if hostname - if Feature.enabled?(:header_read_timeout_buffered_io, default_enabled: :yaml) - gitlab_http = Gitlab::NetHttpAdapter.new(http.address, http.port) + gitlab_http = Gitlab::NetHttpAdapter.new(http.address, http.port) - http.instance_variables.each do |variable| - gitlab_http.instance_variable_set(variable, http.instance_variable_get(variable)) - end - - return gitlab_http + http.instance_variables.each do |variable| + gitlab_http.instance_variable_set(variable, http.instance_variable_get(variable)) end - http + gitlab_http end private diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index d01f7d0074f..8b775d567c8 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -43,27 +43,27 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 46, - 'de' => 15, + 'da_DK' => 44, + 'de' => 14, 'en' => 100, 'eo' => 0, - 'es' => 40, + 'es' => 39, 'fil_PH' => 0, - 'fr' => 11, + 'fr' => 10, 'gl_ES' => 0, 'id_ID' => 0, - 'it' => 2, + 'it' => 1, 'ja' => 34, 'ko' => 12, - 'nb_NO' => 30, + 'nb_NO' => 29, 'nl_NL' => 0, 'pl_PL' => 4, - 'pt_BR' => 49, - 'ro_RO' => 22, - 'ru' => 32, - 'tr_TR' => 14, - 'uk' => 48, - 'zh_CN' => 95, + 'pt_BR' => 50, + 'ro_RO' => 36, + 'ru' => 31, + 'tr_TR' => 13, + 'uk' => 46, + 'zh_CN' => 97, 'zh_HK' => 2, 'zh_TW' => 2 }.freeze diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 3bb34ab2811..74be56df221 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -248,10 +248,9 @@ module Gitlab variable == '%d' ? Random.rand(1000) : Gitlab::Utils.random_string end else - variables.inject({}) do |hash, variable| + variables.each_with_object({}) do |variable, hash| variable_name = variable[/\w+/] hash[variable_name] = Gitlab::Utils.random_string - hash end end end diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index b43d0a0c3eb..e38496ecf67 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -17,11 +17,11 @@ module Gitlab public def initialize(attributes = {}) - @options = OpenStruct.new(attributes) + @options = attributes + end - self.class.instance_eval do - def_delegators :@options, *attributes.keys - end + def method_missing(method, *args) + @options[method] end def execute(current_user, project) diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 7534ab5a9ce..db90886ad11 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -3,19 +3,23 @@ module Gitlab module ImportExport class AvatarSaver + include DurationMeasuring + def initialize(project:, shared:) @project = project @shared = shared end def save - return true unless @project.avatar.exists? + with_duration_measuring do + break true unless @project.avatar.exists? - Gitlab::ImportExport::UploadsManager.new( - project: @project, - shared: @shared, - relative_export_path: 'avatar' - ).save + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared, + relative_export_path: 'avatar' + ).save + end rescue StandardError => e @shared.error(e) false diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2b0467d8779..64ef3dd4830 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -66,7 +66,7 @@ module Gitlab current_size = 0 Gitlab::HTTP.get(url, stream_body: true, allow_object_storage: true) do |fragment| - if [301, 302, 307].include?(fragment.code) + if [301, 302, 303, 307].include?(fragment.code) Gitlab::Import::Logger.warn(message: "received redirect fragment", fragment_code: fragment.code) elsif fragment.code == 200 current_size += fragment.bytesize diff --git a/lib/gitlab/import_export/duration_measuring.rb b/lib/gitlab/import_export/duration_measuring.rb new file mode 100644 index 00000000000..c192be6ae29 --- /dev/null +++ b/lib/gitlab/import_export/duration_measuring.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module DurationMeasuring + extend ActiveSupport::Concern + + included do + attr_reader :duration_s + + def with_duration_measuring + result = nil + + @duration_s = Benchmark.realtime do + result = yield + end + + result + end + end + end + end +end diff --git a/lib/gitlab/import_export/fast_hash_serializer.rb b/lib/gitlab/import_export/fast_hash_serializer.rb index e5d52f945b5..d049609187b 100644 --- a/lib/gitlab/import_export/fast_hash_serializer.rb +++ b/lib/gitlab/import_export/fast_hash_serializer.rb @@ -92,7 +92,7 @@ module Gitlab def simple_serialize subject.as_json( - tree.merge(include: nil, preloads: nil)) + tree.merge(include: nil, preloads: nil, unsafe: true)) end def serialize_includes diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 55b8c1d4531..ebabf537ce5 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -37,7 +37,7 @@ module Gitlab def serialize_root(exportable_path = @exportable_path) attributes = exportable.as_json( - relations_schema.merge(include: nil, preloads: nil)) + relations_schema.merge(include: nil, preloads: nil, unsafe: true)) json_writer.write_attributes(exportable_path, attributes) end @@ -145,8 +145,8 @@ module Gitlab arel_order_classes = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::AREL_ORDER_CLASSES.invert reverse_direction = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_ORDER_DIRECTIONS[direction] reverse_nulls_position = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_NULL_POSITIONS[nulls_position] - order_expression = ::Gitlab::Database.nulls_order(column, direction, nulls_position) - reverse_order_expression = ::Gitlab::Database.nulls_order(column, reverse_direction, reverse_nulls_position) + order_expression = arel_table[column].public_send(direction).public_send(nulls_position) # rubocop:disable GitlabSecurity/PublicSend + reverse_order_expression = arel_table[column].public_send(reverse_direction).public_send(reverse_nulls_position) # rubocop:disable GitlabSecurity/PublicSend ::Gitlab::Pagination::Keyset::Order.build([ ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 47acd49d529..22a7a8dd7cd 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -4,6 +4,7 @@ module Gitlab module ImportExport class LfsSaver include Gitlab::ImportExport::CommandLineUtil + include DurationMeasuring attr_accessor :lfs_json, :project, :shared @@ -16,17 +17,19 @@ module Gitlab end def save - project.lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| - batch.each do |lfs_object| - save_lfs_object(lfs_object) - end + with_duration_measuring do + project.lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| + batch.each do |lfs_object| + save_lfs_object(lfs_object) + end - append_lfs_json_for_batch(batch) - end + append_lfs_json_for_batch(batch) + end - write_lfs_json + write_lfs_json - true + true + end rescue StandardError => e shared.error(e) diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index d3b1bb6a57d..b1f2a17d4b7 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -16,7 +16,7 @@ module Gitlab def map @map ||= begin - @exported_members.inject(missing_keys_tracking_hash) do |hash, member| + @exported_members.each_with_object(missing_keys_tracking_hash) do |member, hash| if member['user'] old_user_id = member['user']['id'] existing_user_id = existing_users_email_map[get_email(member)] @@ -24,8 +24,6 @@ module Gitlab else add_team_member(member) end - - hash end end end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index aafed850afa..63c5afa9595 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -4,6 +4,8 @@ module Gitlab module ImportExport module Project class TreeSaver + include DurationMeasuring + attr_reader :full_path def initialize(project:, current_user:, shared:, params: {}, logger: Gitlab::Import::Logger) @@ -15,9 +17,11 @@ module Gitlab end def save - stream_export + with_duration_measuring do + stream_export - true + true + end rescue StandardError => e @shared.error(e) false diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index fae07039139..454e84bbc04 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -4,6 +4,7 @@ module Gitlab module ImportExport class RepoSaver include Gitlab::ImportExport::CommandLineUtil + include DurationMeasuring attr_reader :exportable, :shared @@ -13,9 +14,12 @@ module Gitlab end def save - return true unless repository_exists? # it's ok to have no repo + with_duration_measuring do + # it's ok to have no repo + break true unless repository_exists? - bundle_to_disk + bundle_to_disk + end end def repository diff --git a/lib/gitlab/import_export/snippets_repo_saver.rb b/lib/gitlab/import_export/snippets_repo_saver.rb index d3b0fe1c18c..ca0d38272e5 100644 --- a/lib/gitlab/import_export/snippets_repo_saver.rb +++ b/lib/gitlab/import_export/snippets_repo_saver.rb @@ -4,6 +4,7 @@ module Gitlab module ImportExport class SnippetsRepoSaver include Gitlab::ImportExport::CommandLineUtil + include DurationMeasuring def initialize(current_user:, project:, shared:) @project = project @@ -12,13 +13,15 @@ module Gitlab end def save - create_snippets_repo_directory + with_duration_measuring do + create_snippets_repo_directory - @project.snippets.find_each.all? do |snippet| - Gitlab::ImportExport::SnippetRepoSaver.new(project: @project, - shared: @shared, - repository: snippet.repository) - .save + @project.snippets.find_each.all? do |snippet| + Gitlab::ImportExport::SnippetRepoSaver.new(project: @project, + shared: @shared, + repository: snippet.repository) + .save + end end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 9f58609fa17..05132fd3edd 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -3,16 +3,20 @@ module Gitlab module ImportExport class UploadsSaver + include DurationMeasuring + def initialize(project:, shared:) @project = project @shared = shared end def save - Gitlab::ImportExport::UploadsManager.new( - project: @project, - shared: @shared - ).save + with_duration_measuring do + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).save + end rescue StandardError => e @shared.error(e) false diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index e8f68f93af0..db5040ec0f6 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -4,17 +4,20 @@ module Gitlab module ImportExport class VersionSaver include Gitlab::ImportExport::CommandLineUtil + include DurationMeasuring def initialize(shared:) @shared = shared end def save - mkdir_p(@shared.export_path) + with_duration_measuring do + mkdir_p(@shared.export_path) - File.write(version_file, Gitlab::ImportExport.version, mode: 'w') - File.write(gitlab_version_file, Gitlab::VERSION, mode: 'w') - File.write(gitlab_revision_file, Gitlab.revision, mode: 'w') + File.write(version_file, Gitlab::ImportExport.version, mode: 'w') + File.write(gitlab_version_file, Gitlab::VERSION, mode: 'w') + File.write(gitlab_revision_file, Gitlab.revision, mode: 'w') + end rescue StandardError => e @shared.error(e) false diff --git a/lib/gitlab/insecure_key_fingerprint.rb b/lib/gitlab/insecure_key_fingerprint.rb index ef342f3819f..43ad64603a6 100644 --- a/lib/gitlab/insecure_key_fingerprint.rb +++ b/lib/gitlab/insecure_key_fingerprint.rb @@ -11,19 +11,12 @@ module Gitlab class InsecureKeyFingerprint attr_accessor :key - alias_attribute :fingerprint_md5, :fingerprint - - # # Gets the base64 encoded string representing a rsa or dsa key # def initialize(key_base64) @key = key_base64 end - def fingerprint - OpenSSL::Digest::MD5.hexdigest(Base64.decode64(@key)).scan(/../).join(':') - end - def fingerprint_sha256 Digest::SHA256.base64digest(Base64.decode64(@key)).scan(/../).join('').delete("=") end diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb index 82c2b3297c1..f347db7bc8c 100644 --- a/lib/gitlab/integrations/sti_type.rb +++ b/lib/gitlab/integrations/sti_type.rb @@ -3,12 +3,12 @@ module Gitlab module Integrations class StiType < ActiveRecord::Type::String - NAMESPACED_INTEGRATIONS = Set.new(%w( + NAMESPACED_INTEGRATIONS = %w[ Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Harbor Irker Jenkins Jira Mattermost MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker Prometheus Pushover Redmine Shimo Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao - )).freeze + ].to_set.freeze def self.namespaced_integrations NAMESPACED_INTEGRATIONS diff --git a/lib/gitlab/lazy.rb b/lib/gitlab/lazy.rb index d7a22aa339e..c589d613efc 100644 --- a/lib/gitlab/lazy.rb +++ b/lib/gitlab/lazy.rb @@ -15,10 +15,10 @@ module Gitlab @block = block end - def method_missing(name, *args, &block) + def method_missing(...) __evaluate__ - @result.__send__(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + @result.__send__(...) # rubocop:disable GitlabSecurity/PublicSend end def respond_to_missing?(name, include_private = false) diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 03655eb7237..89b0f0c802f 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -99,7 +99,7 @@ module Gitlab case actor when DeployKey, Key # Since fingerprint is based on the public key, let's take more bytes from attr_encrypted_db_key_base - actor.fingerprint.delete(':').first(16) + Settings.attr_encrypted_db_key_base_32 + actor.fingerprint_sha256.first(16) + Settings.attr_encrypted_db_key_base_32 when User # Take the last 16 characters as they're more unique than the first 16 actor.id.to_s + actor.encrypted_password.last(16) + Settings.attr_encrypted_db_key_base.first(16) diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index f4984e11c14..51277497c99 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -38,6 +38,10 @@ module Gitlab end end + def full_host + proc { |_env| Settings.gitlab['base_url'] } + end + private def cas3_signout_handler diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb index 065a3a0cf20..8c0f082f61c 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb @@ -120,7 +120,7 @@ module Gitlab .from(array_cte) .join(Arel.sql("LEFT JOIN LATERAL (#{initial_keyset_query.to_sql}) #{table_name} ON TRUE")) - order_by_columns.each { |column| q.where(column.column_expression.not_eq(nil)) } + order_by_columns.each { |c| q.where(c.column_expression.not_eq(nil)) unless c.column.nullable? } q.as('array_scope_lateral_query') end @@ -200,7 +200,7 @@ module Gitlab .project([*order_by_columns.original_column_names_as_arel_string, Arel.sql('position')]) .from("UNNEST(#{list(order_by_columns.array_aggregated_column_names)}) WITH ORDINALITY AS u(#{list(order_by_columns.original_column_names)}, position)") - order_by_columns.each { |column| q.where(Arel.sql(column.original_column_name).not_eq(nil)) } # ignore rows where all columns are NULL + order_by_columns.each { |c| q.where(Arel.sql(c.original_column_name).not_eq(nil)) unless c.column.nullable? } # ignore rows where all columns are NULL q.order(Arel.sql(order_by_without_table_references)).take(1) end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index 1a00692bdbe..290e94401b8 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -99,6 +99,8 @@ module Gitlab field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z') elsif field_value.nil? nil + elsif lower_named_function?(column_definition) + field_value.downcase else field_value.to_s end @@ -184,6 +186,10 @@ module Gitlab private + def lower_named_function?(column_definition) + column_definition.column_expression.is_a?(Arel::Nodes::NamedFunction) && column_definition.column_expression.name&.downcase == 'lower' + end + def composite_row_comparison_possible? !column_definitions.one? && column_definitions.all?(&:not_nullable?) && diff --git a/lib/gitlab/pagination/keyset/simple_order_builder.rb b/lib/gitlab/pagination/keyset/simple_order_builder.rb index 5e79910a3e9..c36bd497aa3 100644 --- a/lib/gitlab/pagination/keyset/simple_order_builder.rb +++ b/lib/gitlab/pagination/keyset/simple_order_builder.rb @@ -11,13 +11,17 @@ module Gitlab # [transformed_scope, true] # true indicates that the new scope was successfully built # [orginal_scope, false] # false indicates that the order values are not supported in this class class SimpleOrderBuilder + NULLS_ORDER_REGEX = /(?<column_name>.*) (?<direction>\bASC\b|\bDESC\b) (?<nullable>\bNULLS LAST\b|\bNULLS FIRST\b)/.freeze + def self.build(scope) new(scope: scope).build end def initialize(scope:) @scope = scope - @order_values = scope.order_values + # We need to run 'compact' because 'nil' is not removed from order_values + # in some cases due to the use of 'default_scope'. + @order_values = scope.order_values.compact @model_class = scope.model @arel_table = @model_class.arel_table @primary_key = @model_class.primary_key @@ -28,10 +32,13 @@ module Gitlab primary_key_descending_order elsif Gitlab::Pagination::Keyset::Order.keyset_aware?(scope) Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) + # Ordered by a primary key. Ex. 'ORDER BY id'. elsif ordered_by_primary_key? primary_key_order + # Ordered by one non-primary table column. Ex. 'ORDER BY created_at'. elsif ordered_by_other_column? column_with_tie_breaker_order + # Ordered by two table columns with the last column as a tie breaker. Ex. 'ORDER BY created, id ASC'. elsif ordered_by_other_column_with_tie_breaker? tie_breaker_attribute = order_values.second @@ -50,6 +57,77 @@ module Gitlab attr_reader :scope, :order_values, :model_class, :arel_table, :primary_key + def table_column?(name) + model_class.column_names.include?(name.to_s) + end + + def primary_key?(attribute) + arel_table[primary_key].to_s == attribute.to_s + end + + def lower_named_function?(attribute) + attribute.is_a?(Arel::Nodes::NamedFunction) && attribute.name&.downcase == 'lower' + end + + def arel_nulls?(order_value) + return unless order_value.is_a?(Arel::Nodes::NullsLast) || order_value.is_a?(Arel::Nodes::NullsFirst) + + column_name = order_value.try(:expr).try(:expr).try(:name) + + table_column?(column_name) + end + + def supported_column?(order_value) + return true if arel_nulls?(order_value) + + attribute = order_value.try(:expr) + return unless attribute + + if lower_named_function?(attribute) + attribute.expressions.one? && attribute.expressions.first.respond_to?(:name) && table_column?(attribute.expressions.first.name) + else + attribute.respond_to?(:name) && table_column?(attribute.name) + end + end + + # This method converts the first order value to a corresponding arel expression + # if the order value uses either NULLS LAST or NULLS FIRST ordering in raw SQL. + # + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/356644 + # We should stop matching raw literals once we switch to using the Arel methods. + def convert_raw_nulls_order! + order_value = order_values.first + + return unless order_value.is_a?(Arel::Nodes::SqlLiteral) + + # Detect NULLS LAST or NULLS FIRST ordering by looking at the raw SQL string. + if matches = order_value.match(NULLS_ORDER_REGEX) + return unless table_column?(matches[:column_name]) + + column_attribute = arel_table[matches[:column_name]] + direction = matches[:direction].downcase.to_sym + nullable = matches[:nullable].downcase.parameterize(separator: '_').to_sym + + # Build an arel order expression for NULLS ordering. + order = direction == :desc ? column_attribute.desc : column_attribute.asc + arel_order_expression = nullable == :nulls_first ? order.nulls_first : order.nulls_last + + order_values[0] = arel_order_expression + end + end + + def nullability(order_value, attribute_name) + nullable = model_class.columns.find { |column| column.name == attribute_name }.null + + if nullable && order_value.is_a?(Arel::Nodes::Ascending) + :nulls_last + elsif nullable && order_value.is_a?(Arel::Nodes::Descending) + :nulls_first + else + :not_nullable + end + end + def primary_key_descending_order Gitlab::Pagination::Keyset::Order.build([ Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( @@ -69,63 +147,76 @@ module Gitlab end def column_with_tie_breaker_order(tie_breaker_column_order = default_tie_breaker_column_order) - order_expression = order_values.first - attribute_name = order_expression.expr.name - - column_nullable = model_class.columns.find { |column| column.name == attribute_name }.null - - nullable = if column_nullable && order_expression.is_a?(Arel::Nodes::Ascending) - :nulls_last - elsif column_nullable && order_expression.is_a?(Arel::Nodes::Descending) - :nulls_first - else - :not_nullable - end - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: attribute_name, - order_expression: order_expression, - nullable: nullable, - distinct: false - ), + column(order_values.first), tie_breaker_column_order ]) end - def ordered_by_primary_key? - return unless order_values.one? + def column(order_value) + return nulls_order_column(order_value) if arel_nulls?(order_value) + return lower_named_function_column(order_value) if lower_named_function?(order_value.expr) - attribute = order_values.first.try(:expr) + attribute_name = order_value.expr.name - return unless attribute + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: attribute_name, + order_expression: order_value, + nullable: nullability(order_value, attribute_name), + distinct: false + ) + end - arel_table[primary_key].to_s == attribute.to_s + def nulls_order_column(order_value) + attribute = order_value.expr.expr + + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: attribute.name, + column_expression: attribute, + order_expression: order_value, + reversed_order_expression: order_value.reverse, + order_direction: order_value.expr.direction, + nullable: order_value.is_a?(Arel::Nodes::NullsLast) ? :nulls_last : :nulls_first, + distinct: false + ) end - def ordered_by_other_column? + def lower_named_function_column(order_value) + attribute_name = order_value.expr.expressions.first.name + + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: attribute_name, + column_expression: Arel::Nodes::NamedFunction.new("LOWER", [model_class.arel_table[attribute_name]]), + order_expression: order_value, + nullable: nullability(order_value, attribute_name), + distinct: false + ) + end + + def ordered_by_primary_key? return unless order_values.one? attribute = order_values.first.try(:expr) + attribute && primary_key?(attribute) + end - return unless attribute - return unless attribute.try(:name) + def ordered_by_other_column? + return unless order_values.one? - model_class.column_names.include?(attribute.name.to_s) + convert_raw_nulls_order! + + supported_column?(order_values.first) end def ordered_by_other_column_with_tie_breaker? return unless order_values.size == 2 - attribute = order_values.first.try(:expr) - tie_breaker_attribute = order_values.second.try(:expr) + convert_raw_nulls_order! - return unless attribute - return unless tie_breaker_attribute - return unless attribute.respond_to?(:name) + return unless supported_column?(order_values.first) - model_class.column_names.include?(attribute.name.to_s) && - arel_table[primary_key].to_s == tie_breaker_attribute.to_s + tie_breaker_attribute = order_values.second.try(:expr) + tie_breaker_attribute && primary_key?(tie_breaker_attribute) end def default_tie_breaker_column_order diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index fca75d1fe01..00304f48dc5 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -11,8 +11,8 @@ module Gitlab @request_context = request_context end - def paginate(relation, exclude_total_headers: false) - paginate_with_limit_optimization(add_default_order(relation)).tap do |data| + def paginate(relation, exclude_total_headers: false, skip_default_order: false) + paginate_with_limit_optimization(add_default_order(relation, skip_default_order: skip_default_order)).tap do |data| add_pagination_headers(data, exclude_total_headers) end end @@ -27,7 +27,6 @@ module Gitlab end return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) - return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops, default_enabled: :yaml) limited_total_count = pagination_data.total_count_with_limit if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT @@ -47,7 +46,9 @@ module Gitlab false end - def add_default_order(relation) + def add_default_order(relation, skip_default_order: false) + return relation if skip_default_order + if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? relation = relation.order(:id) # rubocop: disable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/patch/legacy_database_config.rb b/lib/gitlab/patch/database_config.rb index 6040f737c75..702e8d404b1 100644 --- a/lib/gitlab/patch/legacy_database_config.rb +++ b/lib/gitlab/patch/database_config.rb @@ -28,7 +28,7 @@ module Gitlab module Patch - module LegacyDatabaseConfig + module DatabaseConfig extend ActiveSupport::Concern prepended do @@ -73,23 +73,34 @@ module Gitlab @uses_legacy_database_config = false # rubocop:disable Gitlab/ModuleWithInstanceVariables super.to_h do |env, configs| - # This check is taken from Rails where the transformation - # of a flat database.yml is done into `primary:` - # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/database_configurations.rb#L169 - if configs.is_a?(Hash) && !configs.all? { |_, v| v.is_a?(Hash) } - configs = { "main" => configs } - - @uses_legacy_database_config = true # rubocop:disable Gitlab/ModuleWithInstanceVariables + # TODO: To be removed in 15.0. See https://gitlab.com/gitlab-org/gitlab/-/issues/338182 + # This preload is needed to convert legacy `database.yml` + # from `production: adapter: postgresql` + # into a `production: main: adapter: postgresql` + unless Gitlab::Utils.to_boolean(ENV['SKIP_DATABASE_CONFIG_VALIDATION'], default: false) + # This check is taken from Rails where the transformation + # of a flat database.yml is done into `primary:` + # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/database_configurations.rb#L169 + if configs.is_a?(Hash) && !configs.all? { |_, v| v.is_a?(Hash) } + configs = { "main" => configs } + + @uses_legacy_database_config = true # rubocop:disable Gitlab/ModuleWithInstanceVariables + end end - if Gitlab.ee? && File.exist?(Rails.root.join("config/database_geo.yml")) - migrations_paths = ["ee/db/geo/migrate"] - migrations_paths << "ee/db/geo/post_migrate" unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] + if Gitlab.ee? + if !configs.key?("geo") && File.exist?(Rails.root.join("config/database_geo.yml")) + configs["geo"] = Rails.application.config_for(:database_geo).stringify_keys + end + + if configs.key?("geo") + migrations_paths = Array(configs["geo"]["migrations_paths"]) + migrations_paths << "ee/db/geo/migrate" if migrations_paths.empty? + migrations_paths << "ee/db/geo/post_migrate" unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] - configs["geo"] = - Rails.application.config_for(:database_geo) - .merge(migrations_paths: migrations_paths, schema_migrations_path: "ee/db/geo/schema_migrations") - .stringify_keys + configs["geo"]["migrations_paths"] = migrations_paths.uniq + configs["geo"]["schema_migrations_path"] = "ee/db/geo/schema_migrations" if configs["geo"]["schema_migrations_path"].blank? + end end [env, configs] diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 847f70693f3..e7a12edf763 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -29,7 +29,7 @@ module Gitlab end def project_path - URI.parse(preview).path.sub(%r{\A/}, '') + URI.parse(preview).path.delete_prefix('/') end def uri_encoded_project_path @@ -57,7 +57,7 @@ module Gitlab ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML'), 'https://gitlab.com/pages/plain-html'), ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook'), 'https://gitlab.com/pages/gitbook', 'illustrations/logos/gitbook.svg'), ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'), - ProjectTemplate.new('sse_middleman', 'Static Site Editor/Middleman', _('Middleman project with Static Site Editor support'), 'https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman', 'illustrations/logos/middleman.svg'), + ProjectTemplate.new('middleman', 'Pages/Middleman', _('Everything you need to create a GitLab Pages site using Middleman'), 'https://gitlab.com/gitlab-org/project-templates/middleman', 'illustrations/logos/middleman.svg'), ProjectTemplate.new('gitpod_spring_petclinic', 'Gitpod/Spring Petclinic', _('A Gitpod configured Webapplication in Spring and Java'), 'https://gitlab.com/gitlab-org/project-templates/gitpod-spring-petclinic', 'illustrations/logos/gitpod.svg'), ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'), ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'), diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index e6a73c71e85..4efa29337d1 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -24,7 +24,7 @@ module Gitlab end execution_message do if params[:merge_request_diff_head_sha].blank? - _("Merge request diff sha parameter is required for the merge quick action.") + _("The `/merge` quick action requires the SHA of the head of the branch.") elsif params[:merge_request_diff_head_sha] != quick_action_target.diff_head_sha _("Branch has been updated since the merge was requested.") elsif preferred_strategy = preferred_auto_merge_strategy(quick_action_target) @@ -291,7 +291,7 @@ module Gitlab parse_params do |attention_param| extract_users(attention_param) end - command :attention do |users| + command :attention, :attn do |users| next if users.empty? users.each do |user| diff --git a/lib/gitlab/relative_positioning/item_context.rb b/lib/gitlab/relative_positioning/item_context.rb index 98e52e8e767..ac0598d8d34 100644 --- a/lib/gitlab/relative_positioning/item_context.rb +++ b/lib/gitlab/relative_positioning/item_context.rb @@ -84,7 +84,7 @@ module Gitlab # MAX(relative_position) without the GROUP BY, due to index usage: # https://gitlab.com/gitlab-org/gitlab-foss/issues/54276#note_119340977 relation = scoped_items - .order(Gitlab::Database.nulls_last_order('position', 'DESC')) + .order(Arel.sql('position').desc.nulls_last) .group(grouping_column) .limit(1) @@ -101,7 +101,7 @@ module Gitlab def max_sibling sib = relative_siblings - .order(Gitlab::Database.nulls_last_order('relative_position', 'DESC')) + .order(model_class.arel_table[:relative_position].desc.nulls_last) .first neighbour(sib) @@ -109,7 +109,7 @@ module Gitlab def min_sibling sib = relative_siblings - .order(Gitlab::Database.nulls_last_order('relative_position', 'ASC')) + .order(model_class.arel_table[:relative_position].asc.nulls_last) .first neighbour(sib) diff --git a/lib/gitlab/security/scan_configuration.rb b/lib/gitlab/security/scan_configuration.rb index 381adda7991..14883a34950 100644 --- a/lib/gitlab/security/scan_configuration.rb +++ b/lib/gitlab/security/scan_configuration.rb @@ -31,6 +31,8 @@ module Gitlab def configuration_path; end + def meta_info_path; end + private attr_reader :project, :configured diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index e2df60c46f1..ec514adafc8 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -4,12 +4,24 @@ module Gitlab class Seeder extend ActionView::Helpers::NumberHelper - MASS_INSERT_PROJECT_START = 'mass_insert_project_' - MASS_INSERT_USER_START = 'mass_insert_user_' + MASS_INSERT_PREFIX = 'mass_insert' + MASS_INSERT_PROJECT_START = "#{MASS_INSERT_PREFIX}_project_" + MASS_INSERT_GROUP_START = "#{MASS_INSERT_PREFIX}_group_" + MASS_INSERT_USER_START = "#{MASS_INSERT_PREFIX}_user_" REPORTED_USER_START = 'reported_user_' - ESTIMATED_INSERT_PER_MINUTE = 2_000_000 + ESTIMATED_INSERT_PER_MINUTE = 250_000 MASS_INSERT_ENV = 'MASS_INSERT' + module NamespaceSeed + extend ActiveSupport::Concern + + included do + scope :not_mass_generated, -> do + where.not("path LIKE '#{MASS_INSERT_GROUP_START}%'") + end + end + end + module ProjectSeed extend ActiveSupport::Concern @@ -30,6 +42,10 @@ module Gitlab end end + def self.log_message(message) + puts "#{Time.current}: #{message}" + end + def self.with_mass_insert(size, model) humanized_model_name = model.is_a?(String) ? model : model.model_name.human.pluralize(size) @@ -63,6 +79,7 @@ module Gitlab def self.quiet # Additional seed logic for models. + Namespace.include(NamespaceSeed) Project.include(ProjectSeed) User.include(UserSeed) diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index bc0071f6333..a498e329c3f 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -98,7 +98,7 @@ module Gitlab storages << { name: key, path: storage_paths[key] } end - config = { socket_path: address.sub(/\Aunix:/, '') } + config = { socket_path: address.delete_prefix('unix:') } if Rails.env.test? socket_filename = options[:gitaly_socket] || "gitaly.socket" @@ -124,9 +124,9 @@ module Gitlab config[:storage] = storages - internal_socket_dir = options[:internal_socket_dir] || File.join(gitaly_dir, 'internal_sockets') - FileUtils.mkdir(internal_socket_dir) unless File.exist?(internal_socket_dir) - config[:internal_socket_dir] = internal_socket_dir + runtime_dir = options[:runtime_dir] || File.join(gitaly_dir, 'run') + FileUtils.mkdir(runtime_dir) unless File.exist?(runtime_dir) + config[:runtime_dir] = runtime_dir config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 8a2f3bbe0ee..78682a89655 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -15,16 +15,24 @@ module Gitlab Technology.new(:ed25519_sk, SSHData::PublicKey::SKED25519, [256], %w(sk-ssh-ed25519@openssh.com)) ].freeze + def self.technologies + if Gitlab::FIPS.enabled? + Gitlab::FIPS::SSH_KEY_TECHNOLOGIES + else + TECHNOLOGIES + end + end + def self.technology(name) - TECHNOLOGIES.find { |tech| tech.name.to_s == name.to_s } + technologies.find { |tech| tech.name.to_s == name.to_s } end def self.technology_for_key(key) - TECHNOLOGIES.find { |tech| key.instance_of?(tech.key_class) } + technologies.find { |tech| key.instance_of?(tech.key_class) } end def self.supported_types - TECHNOLOGIES.map(&:name) + technologies.map(&:name) end def self.supported_sizes(name) @@ -32,7 +40,7 @@ module Gitlab end def self.supported_algorithms - TECHNOLOGIES.flat_map { |tech| tech.supported_algorithms } + technologies.flat_map { |tech| tech.supported_algorithms } end def self.supported_algorithms_for_name(name) diff --git a/lib/gitlab/suggestions/commit_message.rb b/lib/gitlab/suggestions/commit_message.rb index 5bca3efe6e1..fcf30cd6df9 100644 --- a/lib/gitlab/suggestions/commit_message.rb +++ b/lib/gitlab/suggestions/commit_message.rb @@ -13,7 +13,7 @@ module Gitlab end def message - project = suggestion_set.project + project = suggestion_set.target_project user_defined_message = @custom_message.presence || project.suggestion_commit_message.presence message = user_defined_message || DEFAULT_SUGGESTION_COMMIT_MESSAGE @@ -37,8 +37,8 @@ module Gitlab 'branch_name' => ->(user, suggestion_set) { suggestion_set.branch }, 'files_count' => ->(user, suggestion_set) { suggestion_set.file_paths.length }, 'file_paths' => ->(user, suggestion_set) { format_paths(suggestion_set.file_paths) }, - 'project_name' => ->(user, suggestion_set) { suggestion_set.project.name }, - 'project_path' => ->(user, suggestion_set) { suggestion_set.project.path }, + 'project_name' => ->(user, suggestion_set) { suggestion_set.target_project.name }, + 'project_path' => ->(user, suggestion_set) { suggestion_set.target_project.path }, 'user_full_name' => ->(user, suggestion_set) { user.name }, 'username' => ->(user, suggestion_set) { user.username }, 'suggestions_count' => ->(user, suggestion_set) { suggestion_set.suggestions.size } diff --git a/lib/gitlab/suggestions/suggestion_set.rb b/lib/gitlab/suggestions/suggestion_set.rb index 53885cdbf19..21a5acf8afe 100644 --- a/lib/gitlab/suggestions/suggestion_set.rb +++ b/lib/gitlab/suggestions/suggestion_set.rb @@ -9,8 +9,12 @@ module Gitlab @suggestions = suggestions end - def project - first_suggestion.project + def source_project + first_suggestion.source_project + end + + def target_project + first_suggestion.target_project end def branch diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 6a98fa12903..54db31ffd6c 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -198,3 +198,4 @@ module Gitlab end end end +# rubocop:enable Rails/Output diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb index 67ecf498cf7..87861b61119 100644 --- a/lib/gitlab/time_tracking_formatter.rb +++ b/lib/gitlab/time_tracking_formatter.rb @@ -8,7 +8,8 @@ module Gitlab CUSTOM_DAY_AND_MONTH_LENGTH = { hours_per_day: 8, days_per_month: 20 }.freeze def parse(string) - string = string.sub(/\A-/, '') + negative_time = string.start_with?('-') + string = string.delete_prefix('-') seconds = begin @@ -19,7 +20,7 @@ module Gitlab nil end - seconds *= -1 if seconds && Regexp.last_match + seconds *= -1 if seconds && negative_time seconds end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index a58b4beb0df..0e7812d08b8 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -15,6 +15,21 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) end + def definition(basename, category: nil, action: nil, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists + definition = YAML.load_file(Rails.root.join("config/events/#{basename}.yml")) + + dispatch_from_definition(definition, label: label, property: property, value: value, context: context, project: project, user: user, namespace: namespace, **extra) + end + + def dispatch_from_definition(definition, **event_data) + definition = definition.with_indifferent_access + + category ||= definition[:category] + action ||= definition[:action] + + event(category, action, **event_data) + end + def options(group) snowplow.options(group) end @@ -39,3 +54,5 @@ module Gitlab end end end + +Gitlab::Tracking.prepend_mod_with('Gitlab::Tracking') diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index fa40a8b678b..e3bf11b00b4 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -71,7 +71,10 @@ module Gitlab url.sub!("#{raw_credentials}@", '') user, _, password = raw_credentials.partition(':') - @credentials ||= { user: user.presence, password: password.presence } + + @credentials ||= {} + @credentials[:user] = user.presence if @credentials[:user].blank? + @credentials[:password] = password.presence if @credentials[:password].blank? end url = Addressable::URI.parse(url) diff --git a/lib/gitlab/usage/service_ping/instrumented_payload.rb b/lib/gitlab/usage/service_ping/instrumented_payload.rb index e04e2e589b2..6cc67321ba1 100644 --- a/lib/gitlab/usage/service_ping/instrumented_payload.rb +++ b/lib/gitlab/usage/service_ping/instrumented_payload.rb @@ -22,7 +22,7 @@ module Gitlab private - # Not all metrics defintions have instrumentation classes + # Not all metrics definitions have instrumentation classes # The value can be computed only for those that have it def instrumented_metrics_defintions Gitlab::Usage::MetricDefinition.with_instrumentation_class diff --git a/lib/gitlab/usage/service_ping_report.rb b/lib/gitlab/usage/service_ping_report.rb index 794f3373043..3e653b186a0 100644 --- a/lib/gitlab/usage/service_ping_report.rb +++ b/lib/gitlab/usage/service_ping_report.rb @@ -18,16 +18,11 @@ module Gitlab private def with_instrumentation_classes(old_payload, output_method) - if Feature.enabled?(:merge_service_ping_instrumented_metrics, default_enabled: :yaml) + instrumented_metrics_key_paths = Gitlab::Usage::ServicePing::PayloadKeysProcessor.new(old_payload).missing_instrumented_metrics_key_paths - instrumented_metrics_key_paths = Gitlab::Usage::ServicePing::PayloadKeysProcessor.new(old_payload).missing_instrumented_metrics_key_paths + instrumented_payload = Gitlab::Usage::ServicePing::InstrumentedPayload.new(instrumented_metrics_key_paths, output_method).build - instrumented_payload = Gitlab::Usage::ServicePing::InstrumentedPayload.new(instrumented_metrics_key_paths, output_method).build - - old_payload.deep_merge(instrumented_payload) - else - old_payload - end + old_payload.deep_merge(instrumented_payload) end def all_metrics_values(cached) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 951ec5ea5c3..b465d4bcc9b 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -70,7 +70,7 @@ module Gitlab def system_usage_data issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)) - counts = { + { counts: { assignee_lists: count(List.assignee), ci_builds: count(::Ci::Build), @@ -166,12 +166,6 @@ module Gitlab data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end } - - if Feature.disabled?(:merge_service_ping_instrumented_metrics, default_enabled: :yaml) - counts[:counts][:boards] = add_metric('CountBoardsMetric', time_frame: 'all') - end - - counts end # rubocop: enable Metrics/AbcSize @@ -513,7 +507,6 @@ module Gitlab { deploy_keys: distinct_count(::DeployKey.where(time_period), :user_id), keys: distinct_count(::Key.regular_keys.where(time_period), :user_id), - merge_requests: distinct_count(::MergeRequest.where(time_period), :author_id), projects_with_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: true))), projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))), remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id), @@ -801,14 +794,9 @@ module Gitlab sent_emails = count(Users::InProductMarketingEmail.group(:track, :series)) clicked_emails = count(Users::InProductMarketingEmail.where.not(cta_clicked_at: nil).group(:track, :series)) - Users::InProductMarketingEmail.tracks.keys.each_with_object({}) do |track, result| + Users::InProductMarketingEmail::ACTIVE_TRACKS.keys.each_with_object({}) do |track, result| + series_amount = Namespaces::InProductMarketingEmailsService.email_count_for_track(track) # rubocop: enable UsageData/LargeTable: - series_amount = - if track.to_sym == Namespaces::InviteTeamEmailService::TRACK - 0 - else - Namespaces::InProductMarketingEmailsService::TRACKS[track.to_sym][:interval_days].count - end 0.upto(series_amount - 1).map do |series| # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`. diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index b8de7de848d..cf3caf3f0c7 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -6,13 +6,18 @@ module Gitlab::UsageDataCounters KNOWN_EVENTS_FILE_PATH = File.expand_path('known_events/ci_templates.yml', __dir__) class << self - def track_unique_project_event(project_id:, template:, config_source:) + def track_unique_project_event(project:, template:, config_source:, user:) expanded_template_name = expand_template_name(template) return unless expanded_template_name Gitlab::UsageDataCounters::HLLRedisCounter.track_event( - ci_template_event_name(expanded_template_name, config_source), values: project_id + ci_template_event_name(expanded_template_name, config_source), values: project.id ) + + namespace = project.namespace + if Feature.enabled?(:route_hll_to_snowplow, namespace, default_enabled: :yaml) + Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, user: user, project: project) + end end def ci_templates(relative_base = 'lib/gitlab/ci/templates') diff --git a/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb new file mode 100644 index 00000000000..8a57a0331b8 --- /dev/null +++ b/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module GitLabCliActivityUniqueCounter + GITLAB_CLI_API_REQUEST_ACTION = 'i_code_review_user_gitlab_cli_api_request' + GITLAB_CLI_USER_AGENT_REGEX = /GitLab\sCLI$/.freeze + + class << self + def track_api_request_when_trackable(user_agent:, user:) + user_agent&.match?(GITLAB_CLI_USER_AGENT_REGEX) && track_unique_action_by_user(GITLAB_CLI_API_REQUEST_ACTION, user) + end + + private + + def track_unique_action_by_user(action, user) + return unless user + + track_unique_action(action, user.id) + end + + def track_unique_action(action, value) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 474ab9a4dd9..3b34cd77cf5 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -81,6 +81,12 @@ module Gitlab track(values, event_name, context: context, time: time) end + # Count unique events for a given time range. + # + # event_names - The list of the events to count. + # start_date - The start date of the time range. + # end_date - The end date of the time range. + # context - Event context, plan level tracking. Available if set when tracking. def unique_events(event_names:, start_date:, end_date:, context: '') count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events| raise SlotMismatch, events unless events_in_same_slot?(events) @@ -100,6 +106,13 @@ module Gitlab known_events.select { |event| event[:category] == category.to_s }.map { |event| event[:name] } end + # Recent 7 or 28 days unique events data for events defined in /lib/gitlab/usage_data_counters/known_events/ + # + # - For metrics for which we store a key per day, we have the last 7 days or last 28 days of data. + # - For metrics for which we store a key per week, we have the last complete week or last 4 complete weeks + # daily or weekly information is in the file we have for events definition /lib/gitlab/usage_data_counters/known_events/ + # - Most of the metrics have weekly aggregation. We recommend this as it generates fewer keys in Redis to store. + # - The aggregation used doesn't affect data granulation. def unique_events_data categories.each_with_object({}) do |category, category_results| events_names = events_for_category(category) diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index a39fa7aca4f..f179f6d679d 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -219,6 +219,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_themekit + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_terraform category: ci_templates redis_slot: ci_templates @@ -615,3 +619,11 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_liquibase + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_matlab + category: ci_templates + redis_slot: ci_templates + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index 42c51ec3921..df2864bba89 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -132,6 +132,11 @@ category: code_review aggregation: weekly feature_flag: usage_data_i_code_review_user_jetbrains_api_request +- name: i_code_review_user_gitlab_cli_api_request + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_gitlab_cli_api_request - name: i_code_review_user_create_mr_from_issue redis_slot: code_review category: code_review @@ -173,62 +178,50 @@ redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_click_single_file_mode_setting redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_click_file_browser_setting redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_click_whitespace_setting redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_diff_view_inline redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_diff_view_parallel redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_file_browser_tree_view redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_file_browser_list_view redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_diff_show_whitespace redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_diff_hide_whitespace redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_diff_single_file redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_diff_multiple_files redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_user_load_conflict_ui redis_slot: code_review category: code_review @@ -241,7 +234,6 @@ redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_diff_searches - name: i_code_review_total_suggestions_applied redis_slot: code_review category: code_review diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index fdf4bc58525..0d89a5181ec 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -25,25 +25,21 @@ redis_slot: edit expiry: 29 aggregation: daily - feature_flag: track_editor_edit_actions - name: g_edit_by_sfe category: ide_edit redis_slot: edit expiry: 29 aggregation: daily - feature_flag: track_editor_edit_actions - name: g_edit_by_sse category: ide_edit redis_slot: edit expiry: 29 aggregation: daily - feature_flag: track_editor_edit_actions - name: g_edit_by_snippet_ide category: ide_edit redis_slot: edit expiry: 29 aggregation: daily - feature_flag: track_editor_edit_actions - name: i_search_total category: search redis_slot: search @@ -343,22 +339,18 @@ redis_slot: secure category: secure aggregation: weekly - feature_flag: users_expanding_widgets_usage_data - name: users_expanding_testing_code_quality_report redis_slot: testing category: testing aggregation: weekly - feature_flag: users_expanding_widgets_usage_data - name: users_expanding_testing_accessibility_report redis_slot: testing category: testing aggregation: weekly - feature_flag: users_expanding_widgets_usage_data - name: users_expanding_testing_license_compliance_report redis_slot: testing category: testing aggregation: weekly - feature_flag: users_expanding_widgets_usage_data - name: users_visiting_testing_license_compliance_full_report redis_slot: testing category: testing diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml index 62b0d6dea86..82787b7bf29 100644 --- a/lib/gitlab/usage_data_counters/known_events/epic_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/epic_events.yml @@ -188,3 +188,33 @@ redis_slot: project_management aggregation: daily feature_flag: track_epics_activity + +- name: g_project_management_epic_related_added + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_related_removed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_blocking_added + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_blocking_removed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_blocked_added + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity diff --git a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml index a56e0a6d370..d80b711f8eb 100644 --- a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml +++ b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml @@ -3,9 +3,7 @@ category: error_tracking redis_slot: error_tracking aggregation: weekly - feature_flag: track_error_tracking_activity - name: error_tracking_view_list category: error_tracking redis_slot: error_tracking aggregation: weekly - feature_flag: track_error_tracking_activity diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index d40ac71afc6..977cc3549d8 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -50,7 +50,7 @@ module Gitlab def alt_usage_data(value = nil, fallback: FALLBACK, &block) if block_given? - { alt_usage_data_block: block.to_s } + { alt_usage_data_block: "non-SQL usage data block" } else { alt_usage_data_value: value } end @@ -58,9 +58,9 @@ module Gitlab def redis_usage_data(counter = nil, &block) if block_given? - { redis_usage_data_block: block.to_s } + { redis_usage_data_block: "non-SQL usage data block" } elsif counter.present? - { redis_usage_data_counter: counter } + { redis_usage_data_counter: counter.to_s } end end @@ -74,6 +74,13 @@ module Gitlab def epics_deepest_relationship_level { epics_deepest_relationship_level: 0 } end + + def topology_usage_data + { + duration_s: 0, + failures: [] + } + end end end end diff --git a/lib/gitlab/utils/delegator_override/validator.rb b/lib/gitlab/utils/delegator_override/validator.rb index 402154b41c2..4449fa75877 100644 --- a/lib/gitlab/utils/delegator_override/validator.rb +++ b/lib/gitlab/utils/delegator_override/validator.rb @@ -28,7 +28,13 @@ module Gitlab end def add_target(target_class) - @target_classes << target_class if target_class + return unless target_class + + @target_classes << target_class + + # Also include all descendants inheriting from the target, + # to make sure we catch methods that are only defined in some of them. + @target_classes += target_class.descendants end # This will make sure allowlist we put into ancestors are all included diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index 3bacad72050..a2d217fb42f 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -11,15 +11,19 @@ module Gitlab include Gitlab::Routing include Gitlab::Allowable - attr_reader :subject + # Presenters should always access the subject through an explicit getter defined with + # `presents ..., as:`, the `__subject__` method is only intended for internal use. + def __subject__ + @subject + end def can?(user, action, overridden_subject = nil) - super(user, action, overridden_subject || subject) + super(user, action, overridden_subject || __subject__) end # delegate all #can? queries to the subject def declarative_policy_delegate - subject + __subject__ end def present(**attributes) @@ -31,15 +35,15 @@ module Gitlab end def is_a?(type) - super || subject.is_a?(type) + super || __subject__.is_a?(type) end def web_url - url_builder.build(subject) + url_builder.build(__subject__) end def web_path - url_builder.build(subject, only_path: true) + url_builder.build(__subject__, only_path: true) end class_methods do @@ -58,7 +62,7 @@ module Gitlab # no-op end - define_method(as) { subject } if as + define_method(as) { __subject__ } if as end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 19d30daa577..d74efd458f6 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -226,6 +226,13 @@ module Gitlab end end + def detect_content_type + [ + Gitlab::Workhorse::DETECT_HEADER, + 'true' + ] + end + protected # This is the outermost encoding of a senddata: header. It is safe for |